Compressed Loading and Saving Matrices

src/TMG-Framework/Loading/LoadMatrixFromCSVMatrix.cs

@@ -17,28 +17,41 @@ You should have received a copy of the GNU General Public License
     along with TMG-Framework for XTMF2.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.IO.Compression;
 using TMG.Utilities;
 
 namespace TMG.Loading;
 
 [Module(Name = "Load Matrix From CSV", Description = "Loads a matrix of data in the shape of the SparseMap from a CSV in third normalized form.",
     DocumentationLink = "http://tmg.utoronto.ca/doc/2.0")]
-public sealed class LoadMatrixFromCSVMatrix : BaseFunction<ReadStream, Matrix>
+public sealed class LoadMatrixFromCSVMatrix : BaseFunction<Matrix>
 {
-    [SubModule(Required = true, Name = "Row Categories", Description = "The sparse map this vector will be shaped in.", Index = 0)]
+    [SubModule(Required = true, Name = "Input", Description = "The input CSV stream.", Index = 0)]
+    public IFunction<ReadStream> Input = null!;
+
+    [SubModule(Required = true, Name = "Row Categories", Description = "The sparse map this vector will be shaped in.", Index = 1)]
     public IFunction<Categories> RowCategories = null!;
 
-    [SubModule(Required = true, Name = "Column Categories", Description = "The sparse map this vector will be shaped in.", Index = 1)]
+    [SubModule(Required = true, Name = "Column Categories", Description = "The sparse map this vector will be shaped in.", Index = 2)]
     public IFunction<Categories> ColumnCategories = null!;
 
-    public override Matrix Invoke(ReadStream stream)
+    [Parameter(Required = true, DefaultValue = "false", Name = "Compressed", Description = "Whether the matrix is compressed.", Index = 3)]
+    public IFunction<bool> Compressed = null!;
+
+    public override Matrix Invoke()
     {
+        var stream = Input.Invoke();
         var columnCategories = ColumnCategories.Invoke();
         var rowCategories = RowCategories.Invoke();
+        var compressed = Compressed.Invoke();
         var ret = new Matrix(rowCategories, columnCategories);
         var flatData = ret.Data;
         var rowSize = columnCategories.Count;
-        using (var reader = new CsvReader(stream, true))
+        using var decompressor = compressed
+            ? new GZipStream(stream, CompressionMode.Decompress, true)
+            : null;
+        Stream inputStream = decompressor is null ? stream : decompressor;
+        using (var reader = new CsvReader(inputStream, true))
         {
             var headers = reader.Headers;
             // read in the destination indexes

src/TMG-Framework/Loading/LoadMatrixFromCSVThirdNormalized.cs

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
     along with TMG-Framework for XTMF2.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.IO.Compression;
 using TMG.Utilities;
 
 namespace TMG.Loading;
@@ -40,6 +41,9 @@ public sealed class LoadMatrixFromCSVThirdNormalized : BaseFunction<ReadStream,
     [Parameter(DefaultValue = "2", Name = "Data Column", Index = 4, Description = "The 0 indexed column containing the data to load index.")]
     public IFunction<int> DataColumn = null!;
 
+    [Parameter(DefaultValue = "false", Name = "Compressed", Index = 5, Description = "Whether the matrix is compressed.")]
+    public IFunction<bool> Compressed = null!;
+
     public override Matrix Invoke(ReadStream stream)
     {
         var rowCategories = RowCategories.Invoke();
@@ -50,12 +54,17 @@ public override Matrix Invoke(ReadStream stream)
         var originColumn = OriginColumn.Invoke();
         var destinationColumn = DestinationColumn.Invoke();
         var dataColumn = DataColumn.Invoke();
+        var compressed = Compressed.Invoke();
         if (originColumn < 0 || destinationColumn < 0 || dataColumn < 0)
         {
             throw new XTMFRuntimeException(this, "Column indexes must be greater than or equal to zero!");
         }
         var minColumnSize = Math.Max(originColumn, dataColumn);
-        using (var reader = new CsvReader(stream, true))
+        using var decompressor = compressed
+            ? new GZipStream(stream, CompressionMode.Decompress, true)
+            : null;
+        Stream inputStream = decompressor is null ? stream : decompressor;
+        using (var reader = new CsvReader(inputStream, true))
         {
             reader.LoadLine();
             while (reader.LoadLine(out var columns))

src/TMG-Framework/Loading/LoadMatrixFromMTX.cs

@@ -17,27 +17,40 @@ You should have received a copy of the GNU General Public License
     along with TMG-Framework for XTMF2.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.IO.Compression;
+
 namespace TMG.Loading;
 
 [Module(Name = "Load SparseMatrix From MTX", Description = "Loads a matrix of data in the shape of the SparseMap from an EMME matrix file.",
 DocumentationLink = "http://tmg.utoronto.ca/doc/2.0")]
-public sealed class LoadMatrixFromMTX : BaseFunction<ReadStream, Matrix>
+public sealed class LoadMatrixFromMTX : BaseFunction<Matrix>
 {
-    [SubModule(Required = true, Name = "Map", Description = "The sparse map this vector will be shaped in.", Index = 0)]
+    [SubModule(Required = true, Name = "Input", Description = "The input MTX stream.", Index = 0)]
+    public IFunction<ReadStream> Input = null!;
+
+    [SubModule(Required = true, Name = "Map", Description = "The sparse map this vector will be shaped in.", Index = 1)]
     public IFunction<Categories> Categories = null!;
 
-    [Parameter(Name = "Convert Between Zone Systems", DefaultValue = "false", Description = "A function that converts between the zone system of the matrix and the zone system of the map.", Index = 1)]
+    [Parameter(Name = "Convert Between Zone Systems", DefaultValue = "false", Description = "A function that converts between the zone system of the matrix and the zone system of the map.", Index = 2)]
     public IFunction<bool> ConvertBetweenZoneSystems = null!;
 
+    [Parameter(Name = "Compressed", DefaultValue = "False", Index = 3,
+        Description = "Is the MTX file compressed (mtx.gz)?")]
+    public IFunction<bool> Compressed = null!;
+
     private const uint MagicNumber = 0xC4D4F1B2;
 
     private const int FloatType = 0x1;
 
-    public override Matrix Invoke(ReadStream context)
+    public override Matrix Invoke()
     {
+        var stream = Input.Invoke();
         var categories = Categories.Invoke();
         var matrix = new Matrix(categories, categories);
-        using (var reader = new BinaryReader(context))
+        Stream readStream = Compressed.Invoke()
+            ? new GZipStream(stream, CompressionMode.Decompress)
+            : stream;
+        using (var reader = new BinaryReader(readStream))
         {
             var magic = reader.ReadUInt32();
             if (magic != MagicNumber)

src/TMG-Framework/Loading/LoadVectorFromCSV.cs

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
     along with TMG-Framework for XTMF2.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.IO.Compression;
 using TMG.Utilities;
 
 namespace TMG.Loading;
@@ -34,19 +35,27 @@ public sealed class LoadVectorFromCSV : BaseFunction<ReadStream, Vector>
     [Parameter(DefaultValue = "1", Name = "Data Column", Index = 2, Description = "The 0 indexed column containing the data to load index.")]
     public IFunction<int> DataColumn = null!;
 
+    [Parameter(DefaultValue = "false", Name = "Compressed", Index = 3, Description = "Whether the vector is compressed.")]
+    public IFunction<bool> Compressed = null!;
+
     public override Vector Invoke(ReadStream stream)
     {
         var map = Categories.Invoke();
         var ret = new Vector(map);
         var data = ret.Data;
         var mapColumn = MapColumn.Invoke();
         var dataColumn = DataColumn.Invoke();
+        var compressed = Compressed.Invoke();
         if (mapColumn < 0 || dataColumn < 0)
         {
             throw new XTMFRuntimeException(this, "Column indexes must be greater than or equal to zero!");
         }
         var minColumnSize = Math.Max(mapColumn, dataColumn);
-        using (var reader = new CsvReader(stream, true))
+        using var decompressor = compressed
+            ? new GZipStream(stream, CompressionMode.Decompress, true)
+            : null;
+        Stream inputStream = decompressor is null ? stream : decompressor;
+        using (var reader = new CsvReader(inputStream, true))
         {
             while (reader.LoadLine(out var columns))
             {

src/TMG-Framework/Saving/SaveMatrixAsCSV.cs

@@ -1,5 +1,5 @@
 /*
-    Copyright 2017 University of Toronto
+    Copyright 2017-2026 University of Toronto
 
     This file is part of TMG-Framework for XTMF2.
 
@@ -17,44 +17,61 @@ You should have received a copy of the GNU General Public License
     along with TMG-Framework for XTMF2.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.IO.Compression;
+
 namespace TMG.Saving;
 
 [Module(Name = "Save Matrix To CSV", Description = "Saves a matrix to the given write stream.",
     DocumentationLink = "http://tmg.utoronto.ca/doc/2.0")]
-public class SaveMatrixAsCSV : BaseAction<(Matrix Matrix, WriteStream Stream)>
+public class SaveMatrixAsCSV : BaseAction
 {
-    [Parameter(Name = "Third Normalized", DefaultValue = "True", Index = 0,
+
+    [SubModule(Required = true, Description = "The matrix to be saved as CSV.", Name = "Matrix", Index = 0)]
+    public IFunction<Matrix> Matrix = null!;
+
+    [SubModule(Required = true, Description = "The write stream to which the CSV will be saved.", Name = "WriteTo", Index = 1)]
+    public IFunction<WriteStream> WriteTo = null!;
+
+    [Parameter(Name = "Third Normalized", DefaultValue = "True", Index = 2,
         Description = "Should the data be saved in third normalized form?  If not it will be saved as a CSV Matrix")]
     public IFunction<bool> ThirdNormalized = null!;
 
-    [Parameter(Name = "First Index Header", DefaultValue = "Origin", Index = 1,
+    [Parameter(Name = "First Index Header", DefaultValue = "Origin", Index = 3,
         Description = "The name of the first index.")]
     public IFunction<string> FirstIndexHeader = null!;
 
-    [Parameter(Name = "Second Index Header", DefaultValue = "Destination", Index = 2,
+    [Parameter(Name = "Second Index Header", DefaultValue = "Destination", Index = 4,
         Description = "The name of the second index.")]
     public IFunction<string> SecondIndexHeader = null!;
 
-    [Parameter(Name = "Data Index Header", DefaultValue = "Data", Index = 3,
+    [Parameter(Name = "Data Index Header", DefaultValue = "Data", Index = 5,
         Description = "The name of the Data index.")]
     public IFunction<string> DataIndexHeader = null!;
 
-    public override void Invoke((Matrix Matrix, WriteStream Stream) context)
+    [Parameter(Name = "Compressed", DefaultValue = "False", Index = 6,
+        Description = "Should the CSV file be compressed (csv.gz)?")]
+    public IFunction<bool> Compressed = null!;
+
+    public override void Invoke()
     {
+        var matrix = Matrix.Invoke();
+        var writeTo = WriteTo.Invoke();
+        Stream outputStream = Compressed.Invoke()
+            ? new GZipStream(writeTo, CompressionMode.Compress)
+            : writeTo;
         if (ThirdNormalized.Invoke())
         {
-            WriteThirdNormalized(context);
+            WriteThirdNormalized(matrix, outputStream);
         }
         else
         {
-            WriteCSVMatrix(context);
+            WriteCSVMatrix(matrix, outputStream);
         }
     }
 
-    private void WriteCSVMatrix((Matrix Matrix, WriteStream Stream) context)
+    private void WriteCSVMatrix(Matrix matrix, Stream stream)
     {
-        using var writer = new StreamWriter(context.Stream);
-        var matrix = context.Matrix;
+        using var writer = new StreamWriter(stream);
         var rowCategories = matrix.RowCategories;
         var columnCategories = matrix.ColumnCategories;
         var rowLength = rowCategories.Count;
@@ -91,32 +108,29 @@ private static void WriteWithQuotes(StreamWriter writer, string name)
         writer.Write('"');
     }
 
-    private void WriteThirdNormalized((Matrix Matrix, WriteStream Stream) context)
+    private void WriteThirdNormalized(Matrix matrix, Stream stream)
     {
-        using (StreamWriter writer = new StreamWriter(context.Stream))
+        using StreamWriter writer = new(stream);
+        var map = matrix.RowCategories;
+        var length = map.Count;
+        var data = matrix.Data;
+        WriteWithQuotes(writer, FirstIndexHeader.Invoke());
+        writer.Write(',');
+        WriteWithQuotes(writer, SecondIndexHeader.Invoke());
+        writer.Write(',');
+        WriteWithQuotes(writer, DataIndexHeader.Invoke());
+        writer.WriteLine();
+        for (int i = 0; i < data.Length; i++)
         {
-            var matrix = context.Matrix;
-            var map = matrix.RowCategories;
-            var length = map.Count;
-            var data = matrix.Data;
-            WriteWithQuotes(writer, FirstIndexHeader.Invoke());
-            writer.Write(',');
-            WriteWithQuotes(writer, SecondIndexHeader.Invoke());
-            writer.Write(',');
-            WriteWithQuotes(writer, DataIndexHeader.Invoke());
-            writer.WriteLine();
-            for (int i = 0; i < data.Length; i++)
+            //ignore zeroed out data to save disk space
+            if (data[i] != 0.0)
             {
-                //ignore zeroed out data to save disk space
-                if (data[i] != 0.0)
-                {
-                    var (row, column) = matrix.GetSparseIndex(i);
-                    writer.Write(row);
-                    writer.Write(',');
-                    writer.Write(column);
-                    writer.Write(',');
-                    writer.WriteLine(data[i]);
-                }
+                var (row, column) = matrix.GetSparseIndex(i);
+                writer.Write(row);
+                writer.Write(',');
+                writer.Write(column);
+                writer.Write(',');
+                writer.WriteLine(data[i]);
             }
         }
     }

src/TMG-Framework/Saving/SaveMatrixAsMTX.cs

@@ -1,5 +1,5 @@
 /*
-    Copyright 2017 University of Toronto
+    Copyright 2017-2026 University of Toronto
 
     This file is part of TMG-Framework for XTMF2.
 
@@ -17,46 +17,60 @@ You should have received a copy of the GNU General Public License
     along with TMG-Framework for XTMF2.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.IO.Compression;
+
 namespace TMG.Saving;
 
 [Module(Name = "Save Matrix To MTX", Description = "Saves a matrix to the given write stream.",
 DocumentationLink = "http://tmg.utoronto.ca/doc/2.0")]
-public class SaveMatrixAsMTX : BaseAction<(Matrix Matrix, WriteStream Stream)>
+public class SaveMatrixAsMTX : BaseAction
 {
     private const uint MagicNumber = 0xC4D4F1B2;
 
     private const int FloatType = 0x1;
 
-    public override void Invoke((Matrix Matrix, WriteStream Stream) context)
+    [SubModule(Required = true, Description = "The matrix to be saved as MTX.", Name = "Matrix", Index = 0)]
+    public IFunction<Matrix> Matrix = null!;
+
+    [SubModule(Required = true, Description = "The write stream to which the MTX will be saved.", Name = "WriteTo", Index = 1)]
+    public IFunction<WriteStream> WriteTo = null!;
+
+    [Parameter(Name = "Compressed", DefaultValue = "False", Index = 2,
+        Description = "Should the MTX file be compressed (mtx.gz)?")]
+    public IFunction<bool> Compressed = null!;
+
+    public override void Invoke()
     {
-        using (var writer = new BinaryWriter(context.Stream))
+        var matrix = Matrix.Invoke();
+        var stream = WriteTo.Invoke();
+        Stream outputStream = Compressed.Invoke()
+            ? new GZipStream(stream, CompressionMode.Compress)
+            : stream;
+        using var writer = new BinaryWriter(outputStream);
+        var rowCategories = matrix.RowCategories;
+        var columnCategories = matrix.ColumnCategories;
+        var rowLength = rowCategories.Count;
+        var columnLength = columnCategories.Count;
+        writer.Write(MagicNumber);
+        writer.Write((int)1);
+        writer.Write(FloatType);
+        writer.Write((int)2);
+        // write out the lengths for each index
+        writer.Write(rowLength);
+        writer.Write(columnLength);
+        writer.Flush();
+        using (var indexWriter = new BinaryWriter(outputStream, Encoding.Unicode, true))
         {
-            var matrix = context.Matrix;
-            var rowCategories = matrix.RowCategories;
-            var columnCategories = matrix.ColumnCategories;
-            var rowLength = rowCategories.Count;
-            var columnLength = columnCategories.Count;
-            writer.Write(MagicNumber);
-            writer.Write((int)1);
-            writer.Write(FloatType);
-            writer.Write((int)2);
-            // write out the lengths for each index
-            writer.Write(rowLength);
-            writer.Write(columnLength);
-            writer.Flush();
-            using (var indexWriter = new BinaryWriter(context.Stream, Encoding.Unicode, true))
+            for (int i = 0; i < rowLength; i++)
+            {
+                indexWriter.Write(rowCategories.GetSparseIndex(i));
+            }
+            for (int i = 0; i < columnLength; i++)
             {
-                for (int i = 0; i < rowLength; i++)
-                {
-                    indexWriter.Write(rowCategories.GetSparseIndex(i));
-                }
-                for (int i = 0; i < columnLength; i++)
-                {
-                    indexWriter.Write(columnCategories.GetSparseIndex(i));
-                }
+                indexWriter.Write(columnCategories.GetSparseIndex(i));
             }
-            // use span to avoid copying the data
-            writer.Write(MemoryMarshal.Cast<float, byte>(matrix.Data));
         }
+        // use span to avoid copying the data
+        writer.Write(MemoryMarshal.Cast<float, byte>(matrix.Data));
     }
 }