Question

We have a library function that goes like this:

class function TFileUtils.ReadTextStream(const AStream: TStream): string;
var
  StringStream: TStringStream;
begin
  StringStream := TStringStream.Create('', TEncoding.Unicode);
  try
    // This is WRONG since CopyFrom might rewind the stream (see Remys comment)
    StringStream.CopyFrom(AStream, AStream.Size - AStream.Position);
    Result := StringStream.DataString;
  finally
    StringStream.Free;
  end;
end;

When I check the string that is returned by the function the first Char is the (little-endian) BOM.

Why doesn’t TStringStream ignore the BOM?

Is there a better way to do this? I don’t need backwards compatibility with older Delphi versions, a working solution for XE2 would be fine.

Answer 1

The BOM has to be coming from the source TStream, as TStringStream does not write a BOM. If you want to ignore the BOM if it is present in the source, you have to do it manually before then copying the data, eg:

class function TFileUtils.ReadTextStream(const AStream: TStream): string;
var
  StreamPos, StreamSize: Int64;
  Buf: TBytes;
  NumBytes: Integer;
  Encoding: TEncoding;
begin
  Result := '';

  StreamPos := AStream.Position;
  StreamSize := AStream.Size - StreamPos;

  // Anything available to read?
  if StreamSize < 1 then Exit;

  // Read the first few bytes from the stream...
  SetLength(Buf, 4);
  NumBytes := AStream.Read(Buf[0], Length(Buf));
  if NumBytes < 1 then Exit;
  Inc(StreamPos, NumBytes);
  Dec(StreamSize, NumBytes);

  // Detect the BOM. If you know for a fact what the TStream data is encoded as, 
  // you can assign the Encoding variable to the appropriate TEncoding object and 
  // GetBufferEncoding() will check for that encoding's BOM only...
  SetLength(Buf, NumBytes);
  Encoding := nil;
  Dec(NumBytes, TEncoding.GetBufferEncoding(Buf, Encoding));

  // If any non-BOM bytes were read than rewind the stream back to that position...
  if NumBytes > 0 then
  begin
    AStream.Seek(-NumBytes, soCurrent);
    Dec(StreamPos, NumBytes);
    Inc(StreamSize, NumBytes);
  end else
  begin
    // Anything left to read after the BOM?
    if StreamSize < 1 then Exit;
  end;

  // Now read and decode whatever is left in the stream...
  StringStream := TStringStream.Create('', Encoding);
  try
    StringStream.CopyFrom(AStream, StreamSize);
    Result := StringStream.DataString;
  finally
    StringStream.Free;
  end;
end;