1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 // Package io provides basic interfaces to I/O primitives.
6 // Its primary job is to wrap existing implementations of such primitives,
7 // such as those in package os, into shared public interfaces that
8 // abstract the functionality, plus some other related primitives.
9 //
10 // Because these interfaces and primitives wrap lower-level operations with
11 // various implementations, unless otherwise informed clients should not
12 // assume they are safe for parallel execution.
13 package io
14 15 import (
16 "errors"
17 "sync"
18 )
19 20 // Seek whence values.
21 const (
22 SeekStart = 0 // seek relative to the origin of the file
23 SeekCurrent = 1 // seek relative to the current offset
24 SeekEnd = 2 // seek relative to the end
25 )
26 27 // ErrShortWrite means that a write accepted fewer bytes than requested
28 // but failed to return an explicit error.
29 var ErrShortWrite = errors.New("short write")
30 31 // errInvalidWrite means that a write returned an impossible count.
32 var errInvalidWrite = errors.New("invalid write result")
33 34 // ErrShortBuffer means that a read required a longer buffer than was provided.
35 var ErrShortBuffer = errors.New("short buffer")
36 37 // EOF is the error returned by Read when no more input is available.
38 // (Read must return EOF itself, not an error wrapping EOF,
39 // because callers will test for EOF using ==.)
40 // Functions should return EOF only to signal a graceful end of input.
41 // If the EOF occurs unexpectedly in a structured data stream,
42 // the appropriate error is either [ErrUnexpectedEOF] or some other error
43 // giving more detail.
44 var EOF = errors.New("EOF")
45 46 // ErrUnexpectedEOF means that EOF was encountered in the
47 // middle of reading a fixed-size block or data structure.
48 var ErrUnexpectedEOF = errors.New("unexpected EOF")
49 50 // ErrNoProgress is returned by some clients of a [Reader] when
51 // many calls to Read have failed to return any data or error,
52 // usually the sign of a broken [Reader] implementation.
53 var ErrNoProgress = errors.New("multiple Read calls return no data or error")
54 55 // Reader is the interface that wraps the basic Read method.
56 //
57 // Read reads up to len(p) bytes into p. It returns the number of bytes
58 // read (0 <= n <= len(p)) and any error encountered. Even if Read
59 // returns n < len(p), it may use all of p as scratch space during the call.
60 // If some data is available but not len(p) bytes, Read conventionally
61 // returns what is available instead of waiting for more.
62 //
63 // When Read encounters an error or end-of-file condition after
64 // successfully reading n > 0 bytes, it returns the number of
65 // bytes read. It may return the (non-nil) error from the same call
66 // or return the error (and n == 0) from a subsequent call.
67 // An instance of this general case is that a Reader returning
68 // a non-zero number of bytes at the end of the input stream may
69 // return either err == EOF or err == nil. The next Read should
70 // return 0, EOF.
71 //
72 // Callers should always process the n > 0 bytes returned before
73 // considering the error err. Doing so correctly handles I/O errors
74 // that happen after reading some bytes and also both of the
75 // allowed EOF behaviors.
76 //
77 // If len(p) == 0, Read should always return n == 0. It may return a
78 // non-nil error if some error condition is known, such as EOF.
79 //
80 // Implementations of Read are discouraged from returning a
81 // zero byte count with a nil error, except when len(p) == 0.
82 // Callers should treat a return of 0 and nil as indicating that
83 // nothing happened; in particular it does not indicate EOF.
84 //
85 // Implementations must not retain p.
86 type Reader interface {
87 Read(p []byte) (n int, err error)
88 }
89 90 // Writer is the interface that wraps the basic Write method.
91 //
92 // Write writes len(p) bytes from p to the underlying data stream.
93 // It returns the number of bytes written from p (0 <= n <= len(p))
94 // and any error encountered that caused the write to stop early.
95 // Write must return a non-nil error if it returns n < len(p).
96 // Write must not modify the slice data, even temporarily.
97 //
98 // Implementations must not retain p.
99 type Writer interface {
100 Write(p []byte) (n int, err error)
101 }
102 103 // Closer is the interface that wraps the basic Close method.
104 //
105 // The behavior of Close after the first call is undefined.
106 // Specific implementations may document their own behavior.
107 type Closer interface {
108 Close() error
109 }
110 111 // Seeker is the interface that wraps the basic Seek method.
112 //
113 // Seek sets the offset for the next Read or Write to offset,
114 // interpreted according to whence:
115 // [SeekStart] means relative to the start of the file,
116 // [SeekCurrent] means relative to the current offset, and
117 // [SeekEnd] means relative to the end
118 // (for example, offset = -2 specifies the penultimate byte of the file).
119 // Seek returns the new offset relative to the start of the
120 // file or an error, if any.
121 //
122 // Seeking to an offset before the start of the file is an error.
123 // Seeking to any positive offset may be allowed, but if the new offset exceeds
124 // the size of the underlying object the behavior of subsequent I/O operations
125 // is implementation-dependent.
126 type Seeker interface {
127 Seek(offset int64, whence int) (int64, error)
128 }
129 130 // ReadWriter is the interface that groups the basic Read and Write methods.
131 type ReadWriter interface {
132 Reader
133 Writer
134 }
135 136 // ReadCloser is the interface that groups the basic Read and Close methods.
137 type ReadCloser interface {
138 Reader
139 Closer
140 }
141 142 // WriteCloser is the interface that groups the basic Write and Close methods.
143 type WriteCloser interface {
144 Writer
145 Closer
146 }
147 148 // ReadWriteCloser is the interface that groups the basic Read, Write and Close methods.
149 type ReadWriteCloser interface {
150 Reader
151 Writer
152 Closer
153 }
154 155 // ReadSeeker is the interface that groups the basic Read and Seek methods.
156 type ReadSeeker interface {
157 Reader
158 Seeker
159 }
160 161 // ReadSeekCloser is the interface that groups the basic Read, Seek and Close
162 // methods.
163 type ReadSeekCloser interface {
164 Reader
165 Seeker
166 Closer
167 }
168 169 // WriteSeeker is the interface that groups the basic Write and Seek methods.
170 type WriteSeeker interface {
171 Writer
172 Seeker
173 }
174 175 // ReadWriteSeeker is the interface that groups the basic Read, Write and Seek methods.
176 type ReadWriteSeeker interface {
177 Reader
178 Writer
179 Seeker
180 }
181 182 // ReaderFrom is the interface that wraps the ReadFrom method.
183 //
184 // ReadFrom reads data from r until EOF or error.
185 // The return value n is the number of bytes read.
186 // Any error except EOF encountered during the read is also returned.
187 //
188 // The [Copy] function uses [ReaderFrom] if available.
189 type ReaderFrom interface {
190 ReadFrom(r Reader) (n int64, err error)
191 }
192 193 // WriterTo is the interface that wraps the WriteTo method.
194 //
195 // WriteTo writes data to w until there's no more data to write or
196 // when an error occurs. The return value n is the number of bytes
197 // written. Any error encountered during the write is also returned.
198 //
199 // The Copy function uses WriterTo if available.
200 type WriterTo interface {
201 WriteTo(w Writer) (n int64, err error)
202 }
203 204 // ReaderAt is the interface that wraps the basic ReadAt method.
205 //
206 // ReadAt reads len(p) bytes into p starting at offset off in the
207 // underlying input source. It returns the number of bytes
208 // read (0 <= n <= len(p)) and any error encountered.
209 //
210 // When ReadAt returns n < len(p), it returns a non-nil error
211 // explaining why more bytes were not returned. In this respect,
212 // ReadAt is stricter than Read.
213 //
214 // Even if ReadAt returns n < len(p), it may use all of p as scratch
215 // space during the call. If some data is available but not len(p) bytes,
216 // ReadAt blocks until either all the data is available or an error occurs.
217 // In this respect ReadAt is different from Read.
218 //
219 // If the n = len(p) bytes returned by ReadAt are at the end of the
220 // input source, ReadAt may return either err == EOF or err == nil.
221 //
222 // If ReadAt is reading from an input source with a seek offset,
223 // ReadAt should not affect nor be affected by the underlying
224 // seek offset.
225 //
226 // Clients of ReadAt can execute parallel ReadAt calls on the
227 // same input source.
228 //
229 // Implementations must not retain p.
230 type ReaderAt interface {
231 ReadAt(p []byte, off int64) (n int, err error)
232 }
233 234 // WriterAt is the interface that wraps the basic WriteAt method.
235 //
236 // WriteAt writes len(p) bytes from p to the underlying data stream
237 // at offset off. It returns the number of bytes written from p (0 <= n <= len(p))
238 // and any error encountered that caused the write to stop early.
239 // WriteAt must return a non-nil error if it returns n < len(p).
240 //
241 // If WriteAt is writing to a destination with a seek offset,
242 // WriteAt should not affect nor be affected by the underlying
243 // seek offset.
244 //
245 // Clients of WriteAt can execute parallel WriteAt calls on the same
246 // destination if the ranges do not overlap.
247 //
248 // Implementations must not retain p.
249 type WriterAt interface {
250 WriteAt(p []byte, off int64) (n int, err error)
251 }
252 253 // ByteReader is the interface that wraps the ReadByte method.
254 //
255 // ReadByte reads and returns the next byte from the input or
256 // any error encountered. If ReadByte returns an error, no input
257 // byte was consumed, and the returned byte value is undefined.
258 //
259 // ReadByte provides an efficient interface for byte-at-time
260 // processing. A [Reader] that does not implement ByteReader
261 // can be wrapped using bufio.NewReader to add this method.
262 type ByteReader interface {
263 ReadByte() (byte, error)
264 }
265 266 // ByteScanner is the interface that adds the UnreadByte method to the
267 // basic ReadByte method.
268 //
269 // UnreadByte causes the next call to ReadByte to return the last byte read.
270 // If the last operation was not a successful call to ReadByte, UnreadByte may
271 // return an error, unread the last byte read (or the byte prior to the
272 // last-unread byte), or (in implementations that support the [Seeker] interface)
273 // seek to one byte before the current offset.
274 type ByteScanner interface {
275 ByteReader
276 UnreadByte() error
277 }
278 279 // ByteWriter is the interface that wraps the WriteByte method.
280 type ByteWriter interface {
281 WriteByte(c byte) error
282 }
283 284 // RuneReader is the interface that wraps the ReadRune method.
285 //
286 // ReadRune reads a single encoded Unicode character
287 // and returns the rune and its size in bytes. If no character is
288 // available, err will be set.
289 type RuneReader interface {
290 ReadRune() (r rune, size int, err error)
291 }
292 293 // RuneScanner is the interface that adds the UnreadRune method to the
294 // basic ReadRune method.
295 //
296 // UnreadRune causes the next call to ReadRune to return the last rune read.
297 // If the last operation was not a successful call to ReadRune, UnreadRune may
298 // return an error, unread the last rune read (or the rune prior to the
299 // last-unread rune), or (in implementations that support the [Seeker] interface)
300 // seek to the start of the rune before the current offset.
301 type RuneScanner interface {
302 RuneReader
303 UnreadRune() error
304 }
305 306 // StringWriter is the interface that wraps the WriteString method.
307 type StringWriter interface {
308 WriteString(s string) (n int, err error)
309 }
310 311 // WriteString writes the contents of the string s to w, which accepts a slice of bytes.
312 // If w implements [StringWriter], [StringWriter.WriteString] is invoked directly.
313 // Otherwise, [Writer.Write] is called exactly once.
314 func WriteString(w Writer, s []byte) (n int, err error) {
315 if sw, ok := w.(StringWriter); ok {
316 return sw.WriteString(s)
317 }
318 return w.Write([]byte(s))
319 }
320 321 // ReadAtLeast reads from r into buf until it has read at least min bytes.
322 // It returns the number of bytes copied and an error if fewer bytes were read.
323 // The error is EOF only if no bytes were read.
324 // If an EOF happens after reading fewer than min bytes,
325 // ReadAtLeast returns [ErrUnexpectedEOF].
326 // If min is greater than the length of buf, ReadAtLeast returns [ErrShortBuffer].
327 // On return, n >= min if and only if err == nil.
328 // If r returns an error having read at least min bytes, the error is dropped.
329 func ReadAtLeast(r Reader, buf []byte, min int) (n int, err error) {
330 if len(buf) < min {
331 return 0, ErrShortBuffer
332 }
333 for n < min && err == nil {
334 var nn int
335 nn, err = r.Read(buf[n:])
336 n += nn
337 }
338 if n >= min {
339 err = nil
340 } else if n > 0 && err == EOF {
341 err = ErrUnexpectedEOF
342 }
343 return
344 }
345 346 // ReadFull reads exactly len(buf) bytes from r into buf.
347 // It returns the number of bytes copied and an error if fewer bytes were read.
348 // The error is EOF only if no bytes were read.
349 // If an EOF happens after reading some but not all the bytes,
350 // ReadFull returns [ErrUnexpectedEOF].
351 // On return, n == len(buf) if and only if err == nil.
352 // If r returns an error having read at least len(buf) bytes, the error is dropped.
353 func ReadFull(r Reader, buf []byte) (n int, err error) {
354 return ReadAtLeast(r, buf, len(buf))
355 }
356 357 // CopyN copies n bytes (or until an error) from src to dst.
358 // It returns the number of bytes copied and the earliest
359 // error encountered while copying.
360 // On return, written == n if and only if err == nil.
361 //
362 // If dst implements [ReaderFrom], the copy is implemented using it.
363 func CopyN(dst Writer, src Reader, n int64) (written int64, err error) {
364 written, err = Copy(dst, LimitReader(src, n))
365 if written == n {
366 return n, nil
367 }
368 if written < n && err == nil {
369 // src stopped early; must have been EOF.
370 err = EOF
371 }
372 return
373 }
374 375 // Copy copies from src to dst until either EOF is reached
376 // on src or an error occurs. It returns the number of bytes
377 // copied and the first error encountered while copying, if any.
378 //
379 // A successful Copy returns err == nil, not err == EOF.
380 // Because Copy is defined to read from src until EOF, it does
381 // not treat an EOF from Read as an error to be reported.
382 //
383 // If src implements [WriterTo],
384 // the copy is implemented by calling src.WriteTo(dst).
385 // Otherwise, if dst implements [ReaderFrom],
386 // the copy is implemented by calling dst.ReadFrom(src).
387 func Copy(dst Writer, src Reader) (written int64, err error) {
388 return copyBuffer(dst, src, nil)
389 }
390 391 // CopyBuffer is identical to Copy except that it stages through the
392 // provided buffer (if one is required) rather than allocating a
393 // temporary one. If buf is nil, one is allocated; otherwise if it has
394 // zero length, CopyBuffer panics.
395 //
396 // If either src implements [WriterTo] or dst implements [ReaderFrom],
397 // buf will not be used to perform the copy.
398 func CopyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) {
399 if buf != nil && len(buf) == 0 {
400 panic("empty buffer in CopyBuffer")
401 }
402 return copyBuffer(dst, src, buf)
403 }
404 405 // copyBuffer is the actual implementation of Copy and CopyBuffer.
406 // if buf is nil, one is allocated.
407 func copyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) {
408 // If the reader has a WriteTo method, use it to do the copy.
409 // Avoids an allocation and a copy.
410 if wt, ok := src.(WriterTo); ok {
411 return wt.WriteTo(dst)
412 }
413 // Similarly, if the writer has a ReadFrom method, use it to do the copy.
414 if rf, ok := dst.(ReaderFrom); ok {
415 return rf.ReadFrom(src)
416 }
417 if buf == nil {
418 size := 32 * 1024
419 if l, ok := src.(*LimitedReader); ok && int64(size) > l.N {
420 if l.N < 1 {
421 size = 1
422 } else {
423 size = int(l.N)
424 }
425 }
426 buf = []byte{:size}
427 }
428 for {
429 nr, er := src.Read(buf)
430 if nr > 0 {
431 nw, ew := dst.Write(buf[0:nr])
432 if nw < 0 || nr < nw {
433 nw = 0
434 if ew == nil {
435 ew = errInvalidWrite
436 }
437 }
438 written += int64(nw)
439 if ew != nil {
440 err = ew
441 break
442 }
443 if nr != nw {
444 err = ErrShortWrite
445 break
446 }
447 }
448 if er != nil {
449 if er != EOF {
450 err = er
451 }
452 break
453 }
454 }
455 return written, err
456 }
457 458 // LimitReader returns a Reader that reads from r
459 // but stops with EOF after n bytes.
460 // The underlying implementation is a *LimitedReader.
461 func LimitReader(r Reader, n int64) Reader { return &LimitedReader{r, n} }
462 463 // A LimitedReader reads from R but limits the amount of
464 // data returned to just N bytes. Each call to Read
465 // updates N to reflect the new amount remaining.
466 // Read returns EOF when N <= 0 or when the underlying R returns EOF.
467 type LimitedReader struct {
468 R Reader // underlying reader
469 N int64 // max bytes remaining
470 }
471 472 func (l *LimitedReader) Read(p []byte) (n int, err error) {
473 if l.N <= 0 {
474 return 0, EOF
475 }
476 if int64(len(p)) > l.N {
477 p = p[0:l.N]
478 }
479 n, err = l.R.Read(p)
480 l.N -= int64(n)
481 return
482 }
483 484 // NewSectionReader returns a [SectionReader] that reads from r
485 // starting at offset off and stops with EOF after n bytes.
486 func NewSectionReader(r ReaderAt, off int64, n int64) *SectionReader {
487 var remaining int64
488 const maxint64 = 1<<63 - 1
489 if off <= maxint64-n {
490 remaining = n + off
491 } else {
492 // Overflow, with no way to return error.
493 // Assume we can read up to an offset of 1<<63 - 1.
494 remaining = maxint64
495 }
496 return &SectionReader{r, off, off, remaining, n}
497 }
498 499 // SectionReader implements Read, Seek, and ReadAt on a section
500 // of an underlying [ReaderAt].
501 type SectionReader struct {
502 r ReaderAt // constant after creation
503 base int64 // constant after creation
504 off int64
505 limit int64 // constant after creation
506 n int64 // constant after creation
507 }
508 509 func (s *SectionReader) Read(p []byte) (n int, err error) {
510 if s.off >= s.limit {
511 return 0, EOF
512 }
513 if max := s.limit - s.off; int64(len(p)) > max {
514 p = p[0:max]
515 }
516 n, err = s.r.ReadAt(p, s.off)
517 s.off += int64(n)
518 return
519 }
520 521 var errWhence = errors.New("Seek: invalid whence")
522 var errOffset = errors.New("Seek: invalid offset")
523 524 func (s *SectionReader) Seek(offset int64, whence int) (int64, error) {
525 switch whence {
526 default:
527 return 0, errWhence
528 case SeekStart:
529 offset += s.base
530 case SeekCurrent:
531 offset += s.off
532 case SeekEnd:
533 offset += s.limit
534 }
535 if offset < s.base {
536 return 0, errOffset
537 }
538 s.off = offset
539 return offset - s.base, nil
540 }
541 542 func (s *SectionReader) ReadAt(p []byte, off int64) (n int, err error) {
543 if off < 0 || off >= s.Size() {
544 return 0, EOF
545 }
546 off += s.base
547 if max := s.limit - off; int64(len(p)) > max {
548 p = p[0:max]
549 n, err = s.r.ReadAt(p, off)
550 if err == nil {
551 err = EOF
552 }
553 return n, err
554 }
555 return s.r.ReadAt(p, off)
556 }
557 558 // Size returns the size of the section in bytes.
559 func (s *SectionReader) Size() int64 { return s.limit - s.base }
560 561 // Outer returns the underlying [ReaderAt] and offsets for the section.
562 //
563 // The returned values are the same that were passed to [NewSectionReader]
564 // when the [SectionReader] was created.
565 func (s *SectionReader) Outer() (r ReaderAt, off int64, n int64) {
566 return s.r, s.base, s.n
567 }
568 569 // An OffsetWriter maps writes at offset base to offset base+off in the underlying writer.
570 type OffsetWriter struct {
571 w WriterAt
572 base int64 // the original offset
573 off int64 // the current offset
574 }
575 576 // NewOffsetWriter returns an [OffsetWriter] that writes to w
577 // starting at offset off.
578 func NewOffsetWriter(w WriterAt, off int64) *OffsetWriter {
579 return &OffsetWriter{w, off, off}
580 }
581 582 func (o *OffsetWriter) Write(p []byte) (n int, err error) {
583 n, err = o.w.WriteAt(p, o.off)
584 o.off += int64(n)
585 return
586 }
587 588 func (o *OffsetWriter) WriteAt(p []byte, off int64) (n int, err error) {
589 if off < 0 {
590 return 0, errOffset
591 }
592 593 off += o.base
594 return o.w.WriteAt(p, off)
595 }
596 597 func (o *OffsetWriter) Seek(offset int64, whence int) (int64, error) {
598 switch whence {
599 default:
600 return 0, errWhence
601 case SeekStart:
602 offset += o.base
603 case SeekCurrent:
604 offset += o.off
605 }
606 if offset < o.base {
607 return 0, errOffset
608 }
609 o.off = offset
610 return offset - o.base, nil
611 }
612 613 // TeeReader returns a [Reader] that writes to w what it reads from r.
614 // All reads from r performed through it are matched with
615 // corresponding writes to w. There is no internal buffering -
616 // the write must complete before the read completes.
617 // Any error encountered while writing is reported as a read error.
618 func TeeReader(r Reader, w Writer) Reader {
619 return &teeReader{r, w}
620 }
621 622 type teeReader struct {
623 r Reader
624 w Writer
625 }
626 627 func (t *teeReader) Read(p []byte) (n int, err error) {
628 n, err = t.r.Read(p)
629 if n > 0 {
630 if n, err := t.w.Write(p[:n]); err != nil {
631 return n, err
632 }
633 }
634 return
635 }
636 637 // Discard is a [Writer] on which all Write calls succeed
638 // without doing anything.
639 var Discard Writer = discard{}
640 641 type discard struct{}
642 643 // discard implements ReaderFrom as an optimization so Copy to
644 // io.Discard can avoid doing unnecessary work.
645 var _ ReaderFrom = discard{}
646 647 func (discard) Write(p []byte) (int, error) {
648 return len(p), nil
649 }
650 651 func (discard) WriteString(s []byte) (int, error) {
652 return len(s), nil
653 }
654 655 var blackHolePool = sync.Pool{
656 New: func() any {
657 b := []byte{:8192}
658 return &b
659 },
660 }
661 662 func (discard) ReadFrom(r Reader) (n int64, err error) {
663 bufp := blackHolePool.Get().(*[]byte)
664 readSize := 0
665 for {
666 readSize, err = r.Read(*bufp)
667 n += int64(readSize)
668 if err != nil {
669 blackHolePool.Put(bufp)
670 if err == EOF {
671 return n, nil
672 }
673 return
674 }
675 }
676 }
677 678 // NopCloser returns a [ReadCloser] with a no-op Close method wrapping
679 // the provided [Reader] r.
680 // If r implements [WriterTo], the returned [ReadCloser] will implement [WriterTo]
681 // by forwarding calls to r.
682 func NopCloser(r Reader) ReadCloser {
683 if _, ok := r.(WriterTo); ok {
684 return nopCloserWriterTo{r}
685 }
686 return nopCloser{r}
687 }
688 689 type nopCloser struct {
690 Reader
691 }
692 693 func (nopCloser) Close() error { return nil }
694 695 type nopCloserWriterTo struct {
696 Reader
697 }
698 699 func (nopCloserWriterTo) Close() error { return nil }
700 701 func (c nopCloserWriterTo) WriteTo(w Writer) (n int64, err error) {
702 return c.Reader.(WriterTo).WriteTo(w)
703 }
704 705 // ReadAll reads from r until an error or EOF and returns the data it read.
706 // A successful call returns err == nil, not err == EOF. Because ReadAll is
707 // defined to read from src until EOF, it does not treat an EOF from Read
708 // as an error to be reported.
709 func ReadAll(r Reader) ([]byte, error) {
710 b := []byte{:0:512}
711 for {
712 n, err := r.Read(b[len(b):cap(b)])
713 b = b[:len(b)+n]
714 if err != nil {
715 if err == EOF {
716 err = nil
717 }
718 return b, err
719 }
720 721 if len(b) == cap(b) {
722 // Add more capacity (let append pick how much).
723 b = append(b, 0)[:len(b)]
724 }
725 }
726 }
727