Fixed Externalize functions to always return allocated buffer, even if source is empty string

.gitignore

@@ -4,5 +4,6 @@
 *.ini
 *.identcache
 *.map
+*.res
 _LOG_.txt
 __history/

README.md

@@ -1,2 +1,9 @@
-# vfs
-Add Virtual File System support to your project. Implement Mods directory support in 2 lines of code
+# VFS (Virtual File System)
+Add Virtual File System support to your project. Implement Mods directory support in 2 lines of code.
+Virtually copies contents of any directory into any directory. Copied contents is available in read-only mode.
+
+## Example:
+```delphi
+VfsImport.MapModsFromListA('D:\Game', 'D:\Game\Mods', 'D:\Game\Mods\list.txt');
+VfsImport.RunVfs(VfsImport.SORT_FIFO);
+```

Vfs.dproj

@@ -30,15 +30,7 @@
     <Borland.Personality>Delphi.Personality</Borland.Personality>
     <Borland.ProjectType>VCLApplication</Borland.ProjectType>
     <BorlandProject>
-<BorlandProject><Delphi.Personality><Parameters><Parameters Name="UseLauncher">False</Parameters><Parameters Name="LoadAllSymbols">True</Parameters><Parameters Name="LoadUnspecifiedSymbols">False</Parameters></Parameters><VersionInfo><VersionInfo Name="IncludeVerInfo">True</VersionInfo><VersionInfo Name="AutoIncBuild">True</VersionInfo><VersionInfo Name="MajorVer">1</VersionInfo><VersionInfo Name="MinorVer">0</VersionInfo><VersionInfo Name="Release">0</VersionInfo><VersionInfo Name="Build">0</VersionInfo><VersionInfo Name="Debug">False</VersionInfo><VersionInfo Name="PreRelease">False</VersionInfo><VersionInfo Name="Special">False</VersionInfo><VersionInfo Name="Private">False</VersionInfo><VersionInfo Name="DLL">False</VersionInfo><VersionInfo Name="Locale">1049</VersionInfo><VersionInfo Name="CodePage">1251</VersionInfo></VersionInfo><VersionInfoKeys><VersionInfoKeys Name="CompanyName">Alexander Shostak (aka EtherniDee)</VersionInfoKeys><VersionInfoKeys Name="FileDescription">Virtual File System</VersionInfoKeys><VersionInfoKeys Name="FileVersion">1.0.0.0</VersionInfoKeys><VersionInfoKeys Name="InternalName"></VersionInfoKeys><VersionInfoKeys Name="LegalCopyright"></VersionInfoKeys><VersionInfoKeys Name="LegalTrademarks"></VersionInfoKeys><VersionInfoKeys Name="OriginalFilename"></VersionInfoKeys><VersionInfoKeys Name="ProductName"></VersionInfoKeys><VersionInfoKeys Name="ProductVersion">1.0.0.0</VersionInfoKeys><VersionInfoKeys Name="Comments"></VersionInfoKeys></VersionInfoKeys><Source><Source Name="MainSource">Vfs.dpr</Source></Source><Excluded_Packages>
-      
-      
-      
-      
-      
-      
-      
-      
+<BorlandProject><Delphi.Personality><Parameters><Parameters Name="UseLauncher">False</Parameters><Parameters Name="LoadAllSymbols">True</Parameters><Parameters Name="LoadUnspecifiedSymbols">False</Parameters></Parameters><VersionInfo><VersionInfo Name="IncludeVerInfo">True</VersionInfo><VersionInfo Name="AutoIncBuild">True</VersionInfo><VersionInfo Name="MajorVer">1</VersionInfo><VersionInfo Name="MinorVer">0</VersionInfo><VersionInfo Name="Release">4</VersionInfo><VersionInfo Name="Build">1</VersionInfo><VersionInfo Name="Debug">False</VersionInfo><VersionInfo Name="PreRelease">False</VersionInfo><VersionInfo Name="Special">False</VersionInfo><VersionInfo Name="Private">False</VersionInfo><VersionInfo Name="DLL">False</VersionInfo><VersionInfo Name="Locale">1049</VersionInfo><VersionInfo Name="CodePage">1251</VersionInfo></VersionInfo><VersionInfoKeys><VersionInfoKeys Name="CompanyName">Alexander Shostak (aka EtherniDee)</VersionInfoKeys><VersionInfoKeys Name="FileDescription">Virtual File System</VersionInfoKeys><VersionInfoKeys Name="FileVersion">1.0.4.1</VersionInfoKeys><VersionInfoKeys Name="InternalName"></VersionInfoKeys><VersionInfoKeys Name="LegalCopyright"></VersionInfoKeys><VersionInfoKeys Name="LegalTrademarks"></VersionInfoKeys><VersionInfoKeys Name="OriginalFilename"></VersionInfoKeys><VersionInfoKeys Name="ProductName"></VersionInfoKeys><VersionInfoKeys Name="ProductVersion">1.0.0.0</VersionInfoKeys><VersionInfoKeys Name="Comments"></VersionInfoKeys></VersionInfoKeys><Source><Source Name="MainSource">Vfs.dpr</Source></Source><Excluded_Packages>
       
       
       

VfsBase.pas

@@ -48,6 +48,9 @@ type
     (* Name in lower case, used for wildcard mask matching *)
     SearchName: WideString;
 
+    (* Absolute path to virtual file/folder location without trailing slash for non-drives *)
+    VirtPath: WideString;
+
     (* Absolute path to real file/folder location without trailing slash for non-drives *)
     RealPath: WideString;
 
@@ -130,6 +133,10 @@ function CallWithoutVfs (Func: TSingleArgExternalFunc; Arg: pointer = nil): inte
    of real and virtual paths is stripped *)
 function GetMappingsReport: WideString;
 
+(* Returns text with all applied mappings on per-file level, separated via #13#10. If ShortenPaths is true, common part
+   of real and virtual paths is stripped *)
+function GetDetailedMappingsReport: WideString;
+
 
 (***)  implementation  (***)
 
@@ -517,6 +524,7 @@ begin
         CopyFileInfoWithoutNames(FileInfo.Base, VfsItem.Info.Base);
       end;
  
+      VfsItem.VirtPath := AbsVirtPath;
       VfsItem.RealPath := AbsRealPath;
       VfsItem.Priority := Priority;
       VfsItem.Attrs    := 0;
@@ -569,7 +577,7 @@ begin
 
     with SysScanDir(AbsRealPath, '*') do begin
       while IterNext(FileInfo.FileName, @FileInfo.Base) do begin
-        if Utils.HasFlag(FileInfo.Base.FileAttributes, Windows.FILE_ATTRIBUTE_DIRECTORY) then begin         
+        if Utils.Flags(FileInfo.Base.FileAttributes).Have(Windows.FILE_ATTRIBUTE_DIRECTORY) then begin         
           if (FileInfo.FileName <> '.') and (FileInfo.FileName <> '..') then begin
             Subdirs.Add(TFileInfo.Create(@FileInfo));
           end;
@@ -693,7 +701,7 @@ begin
   end; // .with
 end; // .function RefreshMappedFile
 
-function GetMappingsReport: WideString;
+function GetMappingsReport_ (Mappings: TList {of TMapping}): WideString;
 const
   COL_PATHS = 0;
   COL_META  = 1;
@@ -811,7 +819,51 @@ begin
   // * * * * * //
   SysUtils.FreeAndNil(Buf);
   SysUtils.FreeAndNil(Line);
-end; // .function GetMappingsReport
+end; // .function GetMappingsReport_
+
+function GetMappingsReport: WideString;
+begin
+  with VfsCritSection do begin
+    Enter;
+    result := GetMappingsReport_(Mappings);
+    Leave;
+  end;
+end;
+
+function CompareMappingsByRealPath (A, B: integer): integer;
+begin
+  result := StrLib.CompareBinStringsW(TMapping(A).AbsRealPath, TMapping(B).AbsRealPath);
+end;
+
+function GetDetailedMappingsReport: WideString;
+var
+{O}  DetailedMappings: {O} TList {of TMapping};
+{Un} VfsItem:          TVfsItem;
+
+begin
+  DetailedMappings := DataLib.NewList(Utils.OWNS_ITEMS);
+  VfsItem          := nil;
+  // * * * * * //
+  with VfsCritSection do begin
+    Enter;
+
+    with DataLib.IterateDict(VfsItems) do begin
+      while IterNext do begin
+        VfsItem := TVfsItem(IterValue);
+
+        // Note, item Attrs is not the same as directory mapping Flags
+        DetailedMappings.Add(TMapping.Make(true, VfsItem.VirtPath, VfsItem.RealPath, false, VfsItem.Attrs));
+      end;
+    end;
+    
+    Leave;
+  end;
+
+  DetailedMappings.CustomSort(CompareMappingsByRealPath);
+  result := GetMappingsReport_(DetailedMappings);
+  // * * * * * //
+  SysUtils.FreeAndNil(DetailedMappings);
+end;
 
 begin
   VfsCritSection.Init;

VfsControl.pas

@@ -37,6 +37,7 @@ type
 function RunVfs (DirListingOrder: VfsBase.TDirListingSortType): LONGBOOL; stdcall;
 var
   CurrDir: WideString;
+  SysDir:  WideString;
 
 begin
   with VfsBase.VfsCritSection do begin
@@ -47,11 +48,16 @@ begin
     if result then begin
       VfsHooks.InstallHooks;
 
-      // Try to ensure, that current directory handle is tracked by VfsOpenFiles
+      // Hask: Try to ensure, that current directory handle is tracked by VfsOpenFiles
+      // Windows SetCurrentDirectoryW is does not reopen directory for the same path, thus
+      // not triggering NtCreateFile
+      // Not thread safe
       CurrDir := WinUtils.GetCurrentDirW;
+      SysDir  := WinUtils.GetSysDirW;
 
-      if CurrDir <> '' then begin
-        WinUtils.SetCurrentDirW(CurrDir);
+      if (CurrDir <> '') and (SysDir <> '') then begin
+        WinUtils.SetCurrentDirW(SysDir);
+        {!} Assert(WinUtils.SetCurrentDirW(CurrDir), 'Failed to restore current directory from system directory during VFS initialization');
       end;
     end;
 
@@ -143,8 +149,8 @@ begin
   AbsRootDir := VfsUtils.NormalizePath(RootDir);
   AbsModsDir := VfsUtils.NormalizePath(ModsDir);
   result     := (AbsRootDir <> '') and (AbsModsDir <> '')  and
-                VfsUtils.GetFileInfo(AbsRootDir, FileInfo) and Utils.HasFlag(Windows.FILE_ATTRIBUTE_DIRECTORY, FileInfo.Base.FileAttributes) and
-                VfsUtils.GetFileInfo(AbsModsDir, FileInfo) and Utils.HasFlag(Windows.FILE_ATTRIBUTE_DIRECTORY, FileInfo.Base.FileAttributes);
+                VfsUtils.GetFileInfo(AbsRootDir, FileInfo) and Utils.Flags(FileInfo.Base.FileAttributes).Have(Windows.FILE_ATTRIBUTE_DIRECTORY) and
+                VfsUtils.GetFileInfo(AbsModsDir, FileInfo) and Utils.Flags(FileInfo.Base.FileAttributes).Have(Windows.FILE_ATTRIBUTE_DIRECTORY);
   
   if result then begin
     ModPathPrefix := VfsUtils.AddBackslash(AbsModsDir);

VfsExport.pas

@@ -24,6 +24,18 @@ exports
 (***)  implementation  (***)
 
 
+function Externalize (const Str: AnsiString): {O} pointer; overload;
+begin
+  GetMem(result, Length(Str) + 1);
+  Utils.CopyMem(Length(Str) + 1, pchar(Str), result);
+end;
+
+function Externalize (const Str: WideString): {O} pointer; overload;
+begin
+  GetMem(result, (Length(Str) + 1) * sizeof(WideChar));
+  Utils.CopyMem((Length(Str) + 1) * sizeof(WideChar), PWideChar(Str), result);
+end;
+
 function MapDir (const VirtPath, RealPath: PWideChar; OverwriteExisting: boolean; Flags: integer = 0): LONGBOOL; stdcall;
 begin
   result := VfsBase.MapDir(WideString(VirtPath), WideString(RealPath), OverwriteExisting, Flags);
@@ -63,25 +75,25 @@ end;
 (* Returns text with all applied mappings, separated via #13#10. If ShortenPaths is true, common part
    of real and virtual paths is stripped. Call MemFree to release result buffer *)
 function GetMappingsReport: {O} PWideChar; stdcall;
-var
-  Res: WideString;
-
 begin
-  result := nil;
-  Res    := VfsBase.GetMappingsReport;
-  GetMem(result, (Length(Res) + 1) * sizeof(WideChar));
-  Utils.CopyMem((Length(Res) + 1) * sizeof(WideChar), PWideChar(Res), result);
+  result := Externalize(VfsBase.GetMappingsReport);
 end;
 
 function GetMappingsReportA: {O} pchar; stdcall;
-var
-  Res: string;
-
 begin
-  result := nil;
-  Res    := VfsBase.GetMappingsReport;
-  GetMem(result, Length(Res) + 1);
-  Utils.CopyMem(Length(Res) + 1, pchar(Res), result);
+  result := Externalize(AnsiString(VfsBase.GetMappingsReport));
+end;
+
+(* Returns text with all applied mappings on per-file level, separated via #13#10. If ShortenPaths is true, common part
+   of real and virtual paths is stripped *)
+function GetDetailedMappingsReport: {O} PWideChar; stdcall;
+begin
+  result := Externalize(VfsBase.GetDetailedMappingsReport);
+end;
+
+function GetDetailedMappingsReportA: {O} pchar; stdcall;
+begin
+  result := Externalize(AnsiString(VfsBase.GetDetailedMappingsReport));
 end;
 
 procedure ConsoleLoggingProc (Operation, Message: pchar); stdcall;
@@ -127,6 +139,8 @@ exports
   RunWatcherA,
   GetMappingsReport,
   GetMappingsReportA,
+  GetDetailedMappingsReport,
+  GetDetailedMappingsReportA,
   MemFree,
   InstallConsoleLogger;
 

VfsHooks.pas

@@ -9,7 +9,7 @@ unit VfsHooks;
 uses
   Windows, SysUtils, Math,
   Utils, WinNative, Concur,
-  StrLib, Alg,
+  StrLib, Alg, WinUtils,
   VfsBase, VfsUtils, VfsPatching,
   VfsDebug, VfsApiDigger, VfsOpenFiles;
 
@@ -31,6 +31,8 @@ var
   NativeNtCreateFile:              WinNative.TNtCreateFile;
   NativeNtClose:                   WinNative.TNtClose;
   NativeNtQueryDirectoryFile:      WinNative.TNtQueryDirectoryFile;
+  NativeNtQueryDirectoryFileEx:    WinNative.TNtQueryDirectoryFileEx;
+
 
   NtQueryAttributesFilePatch:     VfsPatching.TAppliedPatch;
   NtQueryFullAttributesFilePatch: VfsPatching.TAppliedPatch;
@@ -38,6 +40,7 @@ var
   NtCreateFilePatch:              VfsPatching.TAppliedPatch;
   NtClosePatch:                   VfsPatching.TAppliedPatch;
   NtQueryDirectoryFilePatch:      VfsPatching.TAppliedPatch;
+  NtQueryDirectoryFileExPatch:    VfsPatching.TAppliedPatch;
 
 
 (* There is no 100% portable and reliable way to get file path by handle, unless file creation/opening
@@ -48,6 +51,10 @@ var
 function GetFilePathByHandle (hFile: THandle): WideString;
 begin
   result := VfsOpenFiles.GetOpenedFilePath(hFile);
+
+  if (result = '') and VfsDebug.LoggingEnabled then begin
+    WriteLog('GetFilePathByHandle', Format('Failed to get path for handle %x. Current directory is: %s', [integer(hFile), WinUtils.GetCurrentDirW]));
+  end;
 end;
 
 (* Returns single absolute path, not dependant on RootDirectory member. '\??\' prefix is always removed, \\.\ and \\?\ paths remain not touched. *)
@@ -107,7 +114,7 @@ begin
 
   // Return cached VFS file info
   if RedirectedPath <> '' then begin
-    if not HadTrailingDelim or Utils.HasFlag(FILE_ATTRIBUTE_DIRECTORY, FileInfo.Base.FileAttributes) then begin
+    if not HadTrailingDelim or Utils.Flags(FileInfo.Base.FileAttributes).Have(FILE_ATTRIBUTE_DIRECTORY) then begin
       FileInformation.CreationTime   := FileInfo.Base.CreationTime;
       FileInformation.LastAccessTime := FileInfo.Base.LastAccessTime;
       FileInformation.LastWriteTime  := FileInfo.Base.LastWriteTime;
@@ -167,7 +174,7 @@ begin
 
   // Return cached VFS file info
   if RedirectedPath <> '' then begin
-    if not HadTrailingDelim or Utils.HasFlag(FILE_ATTRIBUTE_DIRECTORY, FileInfo.Base.FileAttributes) then begin
+    if not HadTrailingDelim or Utils.Flags(FileInfo.Base.FileAttributes).Have(FILE_ATTRIBUTE_DIRECTORY) then begin
       FileInformation.CreationTime   := FileInfo.Base.CreationTime;
       FileInformation.LastAccessTime := FileInfo.Base.LastAccessTime;
       FileInformation.LastWriteTime  := FileInfo.Base.LastWriteTime;
@@ -539,11 +546,28 @@ begin
   end;
 end; // .function Hook_NtQueryDirectoryFile
 
+function Hook_NtQueryDirectoryFileEx (OrigFunc: WinNative.TNtQueryDirectoryFileEx; FileHandle: HANDLE; Event: HANDLE; ApcRoutine: pointer; ApcContext: PVOID; Io: PIO_STATUS_BLOCK;
+                                      Buffer: PVOID; BufLength: ULONG; InfoClass: integer (* FILE_INFORMATION_CLASS *); QueryFlags: integer; Mask: PUNICODE_STRING): NTSTATUS; stdcall;
+var
+  SingleEntry: LONGBOOL;
+  RestartScan: LONGBOOL;
+
+begin
+  if VfsDebug.LoggingEnabled then begin
+    WriteLog('NtQueryDirectoryFileEx', Format('Handle: %x. QueryFlags: %x', [FileHandle, QueryFlags]));
+  end;
+
+  RestartScan := Utils.Flags(QueryFlags).Have(WinNative.SL_RESTART_SCAN);
+  SingleEntry := Utils.Flags(QueryFlags).Have(WinNative.SL_RETURN_SINGLE_ENTRY);
+  result      := WinNative.NtQueryDirectoryFile(FileHandle, Event, ApcRoutine, ApcContext, Io, Buffer, BufLength, InfoClass, SingleEntry, Mask, RestartScan);
+end;
+
 procedure InstallHooks;
 var
   SetProcessDEPPolicy:        function (dwFlags: integer): LONGBOOL; stdcall;
   hDll:                       Windows.THandle;
   NtdllHandle:                integer;
+  NtQueryDirectoryFileExAddr: WinNative.TNtQueryDirectoryFileEx;
 
 begin
   with HooksCritSection do begin
@@ -618,6 +642,18 @@ begin
         @Hook_NtQueryDirectoryFile,
         @NtQueryDirectoryFilePatch
       );
+
+      NtQueryDirectoryFileExAddr := VfsApiDigger.GetRealProcAddress(NtdllHandle, 'NtQueryDirectoryFileEx');
+
+      if @NtQueryDirectoryFileExAddr <> nil then begin
+        WriteLog('InstallHook', 'Installing NtQueryDirectoryFileEx hook');
+        NativeNtQueryDirectoryFileEx := VfsPatching.SpliceWinApi
+        (
+          @NtQueryDirectoryFileExAddr,
+          @Hook_NtQueryDirectoryFileEx,
+          @NtQueryDirectoryFileExPatch
+        );
+      end;
     end; // .if
 
     Leave;
@@ -635,6 +671,7 @@ begin
     NtCreateFilePatch.Rollback;
     NtClosePatch.Rollback;
     NtQueryDirectoryFilePatch.Rollback;
+    NtQueryDirectoryFileExPatch.Rollback;
 
     Leave;
   end;

VfsImport.pas

@@ -51,6 +51,11 @@ procedure MemFree ({O} Buf: pointer); stdcall; external 'vfs.dll';
 function GetMappingsReport: {O} PWideChar; stdcall; external 'vfs.dll';
 function GetMappingsReportA: {O} pchar; stdcall; external 'vfs.dll';
 
+(* Returns text with all applied mappings on per-file level, separated via #13#10. If ShortenPaths is true, common part
+   of real and virtual paths is stripped *)
+function GetDetailedMappingsReport: {O} PWideChar; stdcall; external 'vfs.dll';
+function GetDetailedMappingsReportA: {O} pchar; stdcall; external 'vfs.dll';
+
 (* Allocates console and install logger, writing messages to console *)
 procedure InstallConsoleLogger; stdcall; external 'vfs.dll';
 

VfsPatching.pas

@@ -159,6 +159,7 @@ begin
   // Create and apply hook at target function start
   p.Clear();
   p.Jump(PatchForge.JMP, SpliceBridge);
+  p.Nop(OverwrittenCodeSize - p.Pos);
 
   if AppliedPatch <> nil then begin
     AppliedPatch.Addr := OrigFunc;

VfsUtils.pas

@@ -506,7 +506,7 @@ var
   FileAttrs: integer;
 
 begin
-  result := GetFileAttrs(Path, FileAttrs) and Utils.HasFlag(Windows.FILE_ATTRIBUTE_DIRECTORY, FileAttrs);
+  result := GetFileAttrs(Path, FileAttrs) and Utils.Flags(FileAttrs).Have(Windows.FILE_ATTRIBUTE_DIRECTORY);
 end;
 
 function SysOpenFile (const NtAbsPath: WideString; {OUT} var Res: Windows.THandle; const OpenMode: TSysOpenFileMode = OPEN_AS_ANY; const AccessMode: ACCESS_MASK = FILE_GENERIC_READ): boolean;

_update_dll.bat

@@ -4,7 +4,7 @@ cls
 set h3dir=D:\Heroes 3
 copy /Y Vfs.dll "%h3dir%\vfs.dll"
 copy /Y Vfs.map "%h3dir%\Vfs.map"
-php "%h3dir%\Tools\ExeMapCompiler\compile.phc" "vfs.map" "./DebugMaps"
+php "%h3dir%\Tools\ExeMapCompiler\compile.phc" "vfs.map" "%h3dir%/DebugMaps"
 echo.
 echo.
 echo %date% %time%