diff --git a/src/TMG-Framework/Loading/LoadMatrixFromCSVMatrix.cs b/src/TMG-Framework/Loading/LoadMatrixFromCSVMatrix.cs index 907f829..b2be489 100644 --- a/src/TMG-Framework/Loading/LoadMatrixFromCSVMatrix.cs +++ b/src/TMG-Framework/Loading/LoadMatrixFromCSVMatrix.cs @@ -17,28 +17,41 @@ You should have received a copy of the GNU General Public License along with TMG-Framework for XTMF2. If not, see . */ +using System.IO.Compression; using TMG.Utilities; namespace TMG.Loading; [Module(Name = "Load Matrix From CSV", Description = "Loads a matrix of data in the shape of the SparseMap from a CSV in third normalized form.", DocumentationLink = "http://tmg.utoronto.ca/doc/2.0")] -public sealed class LoadMatrixFromCSVMatrix : BaseFunction +public sealed class LoadMatrixFromCSVMatrix : BaseFunction { - [SubModule(Required = true, Name = "Row Categories", Description = "The sparse map this vector will be shaped in.", Index = 0)] + [SubModule(Required = true, Name = "Input", Description = "The input CSV stream.", Index = 0)] + public IFunction Input = null!; + + [SubModule(Required = true, Name = "Row Categories", Description = "The sparse map this vector will be shaped in.", Index = 1)] public IFunction RowCategories = null!; - [SubModule(Required = true, Name = "Column Categories", Description = "The sparse map this vector will be shaped in.", Index = 1)] + [SubModule(Required = true, Name = "Column Categories", Description = "The sparse map this vector will be shaped in.", Index = 2)] public IFunction ColumnCategories = null!; - public override Matrix Invoke(ReadStream stream) + [Parameter(Required = true, DefaultValue = "false", Name = "Compressed", Description = "Whether the matrix is compressed.", Index = 3)] + public IFunction Compressed = null!; + + public override Matrix Invoke() { + var stream = Input.Invoke(); var columnCategories = ColumnCategories.Invoke(); var rowCategories = RowCategories.Invoke(); + var compressed = Compressed.Invoke(); var ret = new Matrix(rowCategories, columnCategories); var flatData = ret.Data; var rowSize = columnCategories.Count; - using (var reader = new CsvReader(stream, true)) + using var decompressor = compressed + ? new GZipStream(stream, CompressionMode.Decompress, true) + : null; + Stream inputStream = decompressor is null ? stream : decompressor; + using (var reader = new CsvReader(inputStream, true)) { var headers = reader.Headers; // read in the destination indexes diff --git a/src/TMG-Framework/Loading/LoadMatrixFromCSVThirdNormalized.cs b/src/TMG-Framework/Loading/LoadMatrixFromCSVThirdNormalized.cs index 9088c6b..f304ce2 100644 --- a/src/TMG-Framework/Loading/LoadMatrixFromCSVThirdNormalized.cs +++ b/src/TMG-Framework/Loading/LoadMatrixFromCSVThirdNormalized.cs @@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License along with TMG-Framework for XTMF2. If not, see . */ +using System.IO.Compression; using TMG.Utilities; namespace TMG.Loading; @@ -40,6 +41,9 @@ public sealed class LoadMatrixFromCSVThirdNormalized : BaseFunction DataColumn = null!; + [Parameter(DefaultValue = "false", Name = "Compressed", Index = 5, Description = "Whether the matrix is compressed.")] + public IFunction Compressed = null!; + public override Matrix Invoke(ReadStream stream) { var rowCategories = RowCategories.Invoke(); @@ -50,12 +54,17 @@ public override Matrix Invoke(ReadStream stream) var originColumn = OriginColumn.Invoke(); var destinationColumn = DestinationColumn.Invoke(); var dataColumn = DataColumn.Invoke(); + var compressed = Compressed.Invoke(); if (originColumn < 0 || destinationColumn < 0 || dataColumn < 0) { throw new XTMFRuntimeException(this, "Column indexes must be greater than or equal to zero!"); } var minColumnSize = Math.Max(originColumn, dataColumn); - using (var reader = new CsvReader(stream, true)) + using var decompressor = compressed + ? new GZipStream(stream, CompressionMode.Decompress, true) + : null; + Stream inputStream = decompressor is null ? stream : decompressor; + using (var reader = new CsvReader(inputStream, true)) { reader.LoadLine(); while (reader.LoadLine(out var columns)) diff --git a/src/TMG-Framework/Loading/LoadMatrixFromMTX.cs b/src/TMG-Framework/Loading/LoadMatrixFromMTX.cs index e653a8e..62866ee 100644 --- a/src/TMG-Framework/Loading/LoadMatrixFromMTX.cs +++ b/src/TMG-Framework/Loading/LoadMatrixFromMTX.cs @@ -17,27 +17,40 @@ You should have received a copy of the GNU General Public License along with TMG-Framework for XTMF2. If not, see . */ +using System.IO.Compression; + namespace TMG.Loading; [Module(Name = "Load SparseMatrix From MTX", Description = "Loads a matrix of data in the shape of the SparseMap from an EMME matrix file.", DocumentationLink = "http://tmg.utoronto.ca/doc/2.0")] -public sealed class LoadMatrixFromMTX : BaseFunction +public sealed class LoadMatrixFromMTX : BaseFunction { - [SubModule(Required = true, Name = "Map", Description = "The sparse map this vector will be shaped in.", Index = 0)] + [SubModule(Required = true, Name = "Input", Description = "The input MTX stream.", Index = 0)] + public IFunction Input = null!; + + [SubModule(Required = true, Name = "Map", Description = "The sparse map this vector will be shaped in.", Index = 1)] public IFunction Categories = null!; - [Parameter(Name = "Convert Between Zone Systems", DefaultValue = "false", Description = "A function that converts between the zone system of the matrix and the zone system of the map.", Index = 1)] + [Parameter(Name = "Convert Between Zone Systems", DefaultValue = "false", Description = "A function that converts between the zone system of the matrix and the zone system of the map.", Index = 2)] public IFunction ConvertBetweenZoneSystems = null!; + [Parameter(Name = "Compressed", DefaultValue = "False", Index = 3, + Description = "Is the MTX file compressed (mtx.gz)?")] + public IFunction Compressed = null!; + private const uint MagicNumber = 0xC4D4F1B2; private const int FloatType = 0x1; - public override Matrix Invoke(ReadStream context) + public override Matrix Invoke() { + var stream = Input.Invoke(); var categories = Categories.Invoke(); var matrix = new Matrix(categories, categories); - using (var reader = new BinaryReader(context)) + Stream readStream = Compressed.Invoke() + ? new GZipStream(stream, CompressionMode.Decompress) + : stream; + using (var reader = new BinaryReader(readStream)) { var magic = reader.ReadUInt32(); if (magic != MagicNumber) diff --git a/src/TMG-Framework/Loading/LoadVectorFromCSV.cs b/src/TMG-Framework/Loading/LoadVectorFromCSV.cs index 338b12d..a1fdff8 100644 --- a/src/TMG-Framework/Loading/LoadVectorFromCSV.cs +++ b/src/TMG-Framework/Loading/LoadVectorFromCSV.cs @@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License along with TMG-Framework for XTMF2. If not, see . */ +using System.IO.Compression; using TMG.Utilities; namespace TMG.Loading; @@ -34,6 +35,9 @@ public sealed class LoadVectorFromCSV : BaseFunction [Parameter(DefaultValue = "1", Name = "Data Column", Index = 2, Description = "The 0 indexed column containing the data to load index.")] public IFunction DataColumn = null!; + [Parameter(DefaultValue = "false", Name = "Compressed", Index = 3, Description = "Whether the vector is compressed.")] + public IFunction Compressed = null!; + public override Vector Invoke(ReadStream stream) { var map = Categories.Invoke(); @@ -41,12 +45,17 @@ public override Vector Invoke(ReadStream stream) var data = ret.Data; var mapColumn = MapColumn.Invoke(); var dataColumn = DataColumn.Invoke(); + var compressed = Compressed.Invoke(); if (mapColumn < 0 || dataColumn < 0) { throw new XTMFRuntimeException(this, "Column indexes must be greater than or equal to zero!"); } var minColumnSize = Math.Max(mapColumn, dataColumn); - using (var reader = new CsvReader(stream, true)) + using var decompressor = compressed + ? new GZipStream(stream, CompressionMode.Decompress, true) + : null; + Stream inputStream = decompressor is null ? stream : decompressor; + using (var reader = new CsvReader(inputStream, true)) { while (reader.LoadLine(out var columns)) { diff --git a/src/TMG-Framework/Saving/SaveMatrixAsCSV.cs b/src/TMG-Framework/Saving/SaveMatrixAsCSV.cs index 57714ee..67b90a9 100644 --- a/src/TMG-Framework/Saving/SaveMatrixAsCSV.cs +++ b/src/TMG-Framework/Saving/SaveMatrixAsCSV.cs @@ -1,5 +1,5 @@ /* - Copyright 2017 University of Toronto + Copyright 2017-2026 University of Toronto This file is part of TMG-Framework for XTMF2. @@ -17,44 +17,61 @@ You should have received a copy of the GNU General Public License along with TMG-Framework for XTMF2. If not, see . */ +using System.IO.Compression; + namespace TMG.Saving; [Module(Name = "Save Matrix To CSV", Description = "Saves a matrix to the given write stream.", DocumentationLink = "http://tmg.utoronto.ca/doc/2.0")] -public class SaveMatrixAsCSV : BaseAction<(Matrix Matrix, WriteStream Stream)> +public class SaveMatrixAsCSV : BaseAction { - [Parameter(Name = "Third Normalized", DefaultValue = "True", Index = 0, + + [SubModule(Required = true, Description = "The matrix to be saved as CSV.", Name = "Matrix", Index = 0)] + public IFunction Matrix = null!; + + [SubModule(Required = true, Description = "The write stream to which the CSV will be saved.", Name = "WriteTo", Index = 1)] + public IFunction WriteTo = null!; + + [Parameter(Name = "Third Normalized", DefaultValue = "True", Index = 2, Description = "Should the data be saved in third normalized form? If not it will be saved as a CSV Matrix")] public IFunction ThirdNormalized = null!; - [Parameter(Name = "First Index Header", DefaultValue = "Origin", Index = 1, + [Parameter(Name = "First Index Header", DefaultValue = "Origin", Index = 3, Description = "The name of the first index.")] public IFunction FirstIndexHeader = null!; - [Parameter(Name = "Second Index Header", DefaultValue = "Destination", Index = 2, + [Parameter(Name = "Second Index Header", DefaultValue = "Destination", Index = 4, Description = "The name of the second index.")] public IFunction SecondIndexHeader = null!; - [Parameter(Name = "Data Index Header", DefaultValue = "Data", Index = 3, + [Parameter(Name = "Data Index Header", DefaultValue = "Data", Index = 5, Description = "The name of the Data index.")] public IFunction DataIndexHeader = null!; - public override void Invoke((Matrix Matrix, WriteStream Stream) context) + [Parameter(Name = "Compressed", DefaultValue = "False", Index = 6, + Description = "Should the CSV file be compressed (csv.gz)?")] + public IFunction Compressed = null!; + + public override void Invoke() { + var matrix = Matrix.Invoke(); + var writeTo = WriteTo.Invoke(); + Stream outputStream = Compressed.Invoke() + ? new GZipStream(writeTo, CompressionMode.Compress) + : writeTo; if (ThirdNormalized.Invoke()) { - WriteThirdNormalized(context); + WriteThirdNormalized(matrix, outputStream); } else { - WriteCSVMatrix(context); + WriteCSVMatrix(matrix, outputStream); } } - private void WriteCSVMatrix((Matrix Matrix, WriteStream Stream) context) + private void WriteCSVMatrix(Matrix matrix, Stream stream) { - using var writer = new StreamWriter(context.Stream); - var matrix = context.Matrix; + using var writer = new StreamWriter(stream); var rowCategories = matrix.RowCategories; var columnCategories = matrix.ColumnCategories; var rowLength = rowCategories.Count; @@ -91,32 +108,29 @@ private static void WriteWithQuotes(StreamWriter writer, string name) writer.Write('"'); } - private void WriteThirdNormalized((Matrix Matrix, WriteStream Stream) context) + private void WriteThirdNormalized(Matrix matrix, Stream stream) { - using (StreamWriter writer = new StreamWriter(context.Stream)) + using StreamWriter writer = new(stream); + var map = matrix.RowCategories; + var length = map.Count; + var data = matrix.Data; + WriteWithQuotes(writer, FirstIndexHeader.Invoke()); + writer.Write(','); + WriteWithQuotes(writer, SecondIndexHeader.Invoke()); + writer.Write(','); + WriteWithQuotes(writer, DataIndexHeader.Invoke()); + writer.WriteLine(); + for (int i = 0; i < data.Length; i++) { - var matrix = context.Matrix; - var map = matrix.RowCategories; - var length = map.Count; - var data = matrix.Data; - WriteWithQuotes(writer, FirstIndexHeader.Invoke()); - writer.Write(','); - WriteWithQuotes(writer, SecondIndexHeader.Invoke()); - writer.Write(','); - WriteWithQuotes(writer, DataIndexHeader.Invoke()); - writer.WriteLine(); - for (int i = 0; i < data.Length; i++) + //ignore zeroed out data to save disk space + if (data[i] != 0.0) { - //ignore zeroed out data to save disk space - if (data[i] != 0.0) - { - var (row, column) = matrix.GetSparseIndex(i); - writer.Write(row); - writer.Write(','); - writer.Write(column); - writer.Write(','); - writer.WriteLine(data[i]); - } + var (row, column) = matrix.GetSparseIndex(i); + writer.Write(row); + writer.Write(','); + writer.Write(column); + writer.Write(','); + writer.WriteLine(data[i]); } } } diff --git a/src/TMG-Framework/Saving/SaveMatrixAsMTX.cs b/src/TMG-Framework/Saving/SaveMatrixAsMTX.cs index 92ef800..9c7cb22 100644 --- a/src/TMG-Framework/Saving/SaveMatrixAsMTX.cs +++ b/src/TMG-Framework/Saving/SaveMatrixAsMTX.cs @@ -1,5 +1,5 @@ /* - Copyright 2017 University of Toronto + Copyright 2017-2026 University of Toronto This file is part of TMG-Framework for XTMF2. @@ -17,46 +17,60 @@ You should have received a copy of the GNU General Public License along with TMG-Framework for XTMF2. If not, see . */ +using System.IO.Compression; + namespace TMG.Saving; [Module(Name = "Save Matrix To MTX", Description = "Saves a matrix to the given write stream.", DocumentationLink = "http://tmg.utoronto.ca/doc/2.0")] -public class SaveMatrixAsMTX : BaseAction<(Matrix Matrix, WriteStream Stream)> +public class SaveMatrixAsMTX : BaseAction { private const uint MagicNumber = 0xC4D4F1B2; private const int FloatType = 0x1; - public override void Invoke((Matrix Matrix, WriteStream Stream) context) + [SubModule(Required = true, Description = "The matrix to be saved as MTX.", Name = "Matrix", Index = 0)] + public IFunction Matrix = null!; + + [SubModule(Required = true, Description = "The write stream to which the MTX will be saved.", Name = "WriteTo", Index = 1)] + public IFunction WriteTo = null!; + + [Parameter(Name = "Compressed", DefaultValue = "False", Index = 2, + Description = "Should the MTX file be compressed (mtx.gz)?")] + public IFunction Compressed = null!; + + public override void Invoke() { - using (var writer = new BinaryWriter(context.Stream)) + var matrix = Matrix.Invoke(); + var stream = WriteTo.Invoke(); + Stream outputStream = Compressed.Invoke() + ? new GZipStream(stream, CompressionMode.Compress) + : stream; + using var writer = new BinaryWriter(outputStream); + var rowCategories = matrix.RowCategories; + var columnCategories = matrix.ColumnCategories; + var rowLength = rowCategories.Count; + var columnLength = columnCategories.Count; + writer.Write(MagicNumber); + writer.Write((int)1); + writer.Write(FloatType); + writer.Write((int)2); + // write out the lengths for each index + writer.Write(rowLength); + writer.Write(columnLength); + writer.Flush(); + using (var indexWriter = new BinaryWriter(outputStream, Encoding.Unicode, true)) { - var matrix = context.Matrix; - var rowCategories = matrix.RowCategories; - var columnCategories = matrix.ColumnCategories; - var rowLength = rowCategories.Count; - var columnLength = columnCategories.Count; - writer.Write(MagicNumber); - writer.Write((int)1); - writer.Write(FloatType); - writer.Write((int)2); - // write out the lengths for each index - writer.Write(rowLength); - writer.Write(columnLength); - writer.Flush(); - using (var indexWriter = new BinaryWriter(context.Stream, Encoding.Unicode, true)) + for (int i = 0; i < rowLength; i++) + { + indexWriter.Write(rowCategories.GetSparseIndex(i)); + } + for (int i = 0; i < columnLength; i++) { - for (int i = 0; i < rowLength; i++) - { - indexWriter.Write(rowCategories.GetSparseIndex(i)); - } - for (int i = 0; i < columnLength; i++) - { - indexWriter.Write(columnCategories.GetSparseIndex(i)); - } + indexWriter.Write(columnCategories.GetSparseIndex(i)); } - // use span to avoid copying the data - writer.Write(MemoryMarshal.Cast(matrix.Data)); } + // use span to avoid copying the data + writer.Write(MemoryMarshal.Cast(matrix.Data)); } } diff --git a/src/TMG-Framework/Saving/SaveVectorAsCSV.cs b/src/TMG-Framework/Saving/SaveVectorAsCSV.cs index 257a1d1..63db084 100644 --- a/src/TMG-Framework/Saving/SaveVectorAsCSV.cs +++ b/src/TMG-Framework/Saving/SaveVectorAsCSV.cs @@ -17,6 +17,8 @@ You should have received a copy of the GNU General Public License along with TMG-Framework for XTMF2. If not, see . */ +using System.IO.Compression; + namespace TMG.Saving; [Module(Name = "Save Vector To CSV", Description = "Saves a vector to the given write stream.", @@ -25,7 +27,10 @@ public sealed class SaveVectorAsCSV : BaseAction<(Vector Data, WriteStream Strea { public override void Invoke((Vector Data, WriteStream Stream) context) { - using (var writer = new StreamWriter(context.Stream)) + Stream outputStream = Compressed.Invoke() + ? new GZipStream(context.Stream, CompressionMode.Compress) + : context.Stream; + using (var writer = new StreamWriter(outputStream)) { var data = context.Data; var flatData = data.Data; @@ -47,6 +52,9 @@ public override void Invoke((Vector Data, WriteStream Stream) context) [Parameter(Name = "Data Column Name", DefaultValue = "Data", Description = "The header to give to the data", Index = 1)] public IFunction DataColumnName = null!; + [Parameter(Name = "Compressed", DefaultValue = "False", Description = "Should the CSV file be compressed (csv.gz)?", Index = 2)] + public IFunction Compressed = null!; + private void WriteHeaders(StreamWriter writer) { writer.Write('\"'); diff --git a/tests/TMG-Framework.Test/Loading/TestLoadMatrix.cs b/tests/TMG-Framework.Test/Loading/TestLoadMatrix.cs index 48c82ff..3d30b6c 100644 --- a/tests/TMG-Framework.Test/Loading/TestLoadMatrix.cs +++ b/tests/TMG-Framework.Test/Loading/TestLoadMatrix.cs @@ -42,6 +42,7 @@ public void LoadMatrixFromCSVMatrix() var matrixFileName = MatrixHelper.WriteMatrixToCSVMatrix(map, data); var matrixLoader = new LoadMatrixFromCSVMatrix() { + Compressed = Helper.CreateParameter(false), ColumnCategories = Helper.CreateParameter(map), RowCategories = Helper.CreateParameter(map) }; @@ -50,7 +51,8 @@ public void LoadMatrixFromCSVMatrix() FilePath = Helper.CreateParameter(matrixFileName) }).Invoke()) { - var vector = matrixLoader.Invoke(stream); + matrixLoader.Input = Helper.CreateParameter(stream); + var vector = matrixLoader.Invoke(); Assert.AreSame(map, vector.RowCategories); var vData = vector.Data; for (int i = 0; i < map.Count; i++) @@ -86,7 +88,8 @@ public void LoadMatrixFromCSVThirdNormalized() ColumnCategories = Helper.CreateParameter(map), OriginColumn = Helper.CreateParameter(0), DestinationColumn = Helper.CreateParameter(0), - DataColumn = Helper.CreateParameter(0) + DataColumn = Helper.CreateParameter(0), + Compressed = Helper.CreateParameter(false) }; using (var stream = (new OpenReadStreamFromFile() { @@ -123,30 +126,33 @@ public void TestLoadMatrixFromMTX() } } var matrixFileName = MatrixHelper.WriteMatrixToMTX(map, data); + using var stream = new OpenReadStreamFromFile() + { + FilePath = Helper.CreateParameter(matrixFileName) + }.Invoke(); + var matrixLoader = new LoadMatrixFromMTX() { Categories = Helper.CreateParameter(map), - ConvertBetweenZoneSystems = Helper.CreateParameter(false) + ConvertBetweenZoneSystems = Helper.CreateParameter(false), + Compressed = Helper.CreateParameter(false), + Input = Helper.CreateParameter(stream), }; - using (var stream = (new OpenReadStreamFromFile() + + var vector = matrixLoader.Invoke(); + Assert.AreSame(map, vector.RowCategories); + var vData = vector.Data; + for (int i = 0; i < map.Count; i++) { - FilePath = Helper.CreateParameter(matrixFileName) - }).Invoke()) - { - var vector = matrixLoader.Invoke(stream); - Assert.AreSame(map, vector.RowCategories); - var vData = vector.Data; - for (int i = 0; i < map.Count; i++) + for (int j = 0; j < map.Count; j++) { - for (int j = 0; j < map.Count; j++) + if (Math.Abs((2.0f + i * j) - vData[i * map.Count + j]) < 0.00001f) { - if (Math.Abs((2.0f + i * j) - vData[i * map.Count + j]) < 0.00001f) - { - Assert.AreEqual(2.0f + i * j, vData[i * map.Count + j], 0.00001f); - } + Assert.AreEqual(2.0f + i * j, vData[i * map.Count + j], 0.00001f); } } } + } [TestMethod] @@ -164,29 +170,32 @@ public void TestLoadMatrixFromMTXDifferentZoneSystem() } } var matrixFileName = MatrixHelper.WriteMatrixToMTX(bigMap, data); + using var stream = new OpenReadStreamFromFile() + { + FilePath = Helper.CreateParameter(matrixFileName) + }.Invoke(); + var matrixLoader = new LoadMatrixFromMTX() { + Input = Helper.CreateParameter(stream), Categories = Helper.CreateParameter(smallMap), - ConvertBetweenZoneSystems = Helper.CreateParameter(true) + ConvertBetweenZoneSystems = Helper.CreateParameter(true), + Compressed = Helper.CreateParameter(false), }; - using (var stream = (new OpenReadStreamFromFile() - { - FilePath = Helper.CreateParameter(matrixFileName) - }).Invoke()) + + var matrix = matrixLoader.Invoke(); + Assert.AreSame(smallMap, matrix.RowCategories); + var vData = matrix.Data; + for (int i = 0; i < smallMap.Count; i++) { - var matrix = matrixLoader.Invoke(stream); - Assert.AreSame(smallMap, matrix.RowCategories); - var vData = matrix.Data; - for (int i = 0; i < smallMap.Count; i++) + for (int j = 0; j < smallMap.Count; j++) { - for (int j = 0; j < smallMap.Count; j++) + if (Math.Abs((2.0f + i * j) - vData[i * smallMap.Count + j]) < 0.00001f) { - if (Math.Abs((2.0f + i * j) - vData[i * smallMap.Count + j]) < 0.00001f) - { - Assert.AreEqual(2.0f + i * j, vData[i * smallMap.Count + j], 0.00001f); - } + Assert.AreEqual(2.0f + i * j, vData[i * smallMap.Count + j], 0.00001f); } } } + } } diff --git a/tests/TMG-Framework.Test/Loading/TestLoadVector.cs b/tests/TMG-Framework.Test/Loading/TestLoadVector.cs index 4793e1a..043a86d 100644 --- a/tests/TMG-Framework.Test/Loading/TestLoadVector.cs +++ b/tests/TMG-Framework.Test/Loading/TestLoadVector.cs @@ -39,7 +39,8 @@ public void TestLoadVectorFromCSV() { Categories = Helper.CreateParameter(map), MapColumn = Helper.CreateParameter(0), - DataColumn = Helper.CreateParameter(1) + DataColumn = Helper.CreateParameter(1), + Compressed = Helper.CreateParameter(false) }; using (var stream = Helper.CreateReadStreamFromFile(vectorFileName)) { @@ -67,7 +68,8 @@ internal static string WriteVectorCSV(Categories categories, float[] data) var save = new TMG.Saving.SaveVectorAsCSV() { MapColumnName = Helper.CreateParameter("Zone"), - DataColumnName = Helper.CreateParameter("Data") + DataColumnName = Helper.CreateParameter("Data"), + Compressed = Helper.CreateParameter(false) }; using (var writeStream = Helper.CreateWriteStreamFromFile(fileName)) { diff --git a/tests/TMG-Framework.Test/Saving/TestSaveMatrixAsCSV.cs b/tests/TMG-Framework.Test/Saving/TestSaveMatrixAsCSV.cs index aa44733..d63c220 100644 --- a/tests/TMG-Framework.Test/Saving/TestSaveMatrixAsCSV.cs +++ b/tests/TMG-Framework.Test/Saving/TestSaveMatrixAsCSV.cs @@ -33,16 +33,19 @@ public void SaveMatrixAsCSVAsMatrix() FirstIndexHeader = Helper.CreateParameter("First"), SecondIndexHeader = Helper.CreateParameter("Second"), DataIndexHeader = Helper.CreateParameter("Data"), - ThirdNormalized = Helper.CreateParameter(false) + ThirdNormalized = Helper.CreateParameter(false), + Compressed = Helper.CreateParameter(false) }; var categories = MapHelper.LoadMap(MapHelper.WriteCSV(2000)); var a = new Matrix(categories, categories); + module.Matrix = Helper.CreateParameter(a); var fileName = Path.GetTempFileName(); try { using (var writeStream = Helper.CreateWriteStreamFromFile(fileName)) { - module.Invoke((a, writeStream)); + module.WriteTo = Helper.CreateParameter(writeStream); + module.Invoke(); } } finally @@ -63,25 +66,30 @@ public void SaveMatrixAsCSVAsMatrixAndLoad() FirstIndexHeader = Helper.CreateParameter("First"), SecondIndexHeader = Helper.CreateParameter("Second"), DataIndexHeader = Helper.CreateParameter("Data"), - ThirdNormalized = Helper.CreateParameter(false) + ThirdNormalized = Helper.CreateParameter(false), + Compressed = Helper.CreateParameter(true) }; var categories = MapHelper.LoadMap(MapHelper.WriteCSV(2000)); var a = new Matrix(categories, categories); + module.Matrix = Helper.CreateParameter(a); var fileName = Path.GetTempFileName(); try { using (var writeStream = Helper.CreateWriteStreamFromFile(fileName)) { - module.Invoke((a, writeStream)); + module.WriteTo = Helper.CreateParameter(writeStream); + module.Invoke(); } // Now that the matrix has been saved attempt to load it back in. using (var readStream = Helper.CreateReadStreamFromFile(fileName)) { var readMatrix = new TMG.Loading.LoadMatrixFromCSVMatrix() { + Input = Helper.CreateParameter(readStream), RowCategories = Helper.CreateParameter(categories), - ColumnCategories = Helper.CreateParameter(categories) - }.Invoke(readStream); + ColumnCategories = Helper.CreateParameter(categories), + Compressed = Helper.CreateParameter(true) + }.Invoke(); string? error = null; Assert.IsTrue(MatrixHelper.Compare(a, readMatrix, ref error), error); } @@ -104,16 +112,19 @@ public void SaveMatrixAsCSVAsThirdNormalized() FirstIndexHeader = Helper.CreateParameter("First"), SecondIndexHeader = Helper.CreateParameter("Second"), DataIndexHeader = Helper.CreateParameter("Data"), - ThirdNormalized = Helper.CreateParameter(true) + ThirdNormalized = Helper.CreateParameter(true), + Compressed = Helper.CreateParameter(false) }; var categories = MapHelper.LoadMap(MapHelper.WriteCSV(2000)); var a = new Matrix(categories, categories); + module.Matrix = Helper.CreateParameter(a); var fileName = Path.GetTempFileName(); try { using (var writeStream = Helper.CreateWriteStreamFromFile(fileName)) { - module.Invoke((a, writeStream)); + module.WriteTo = Helper.CreateParameter(writeStream); + module.Invoke(); } } finally @@ -134,16 +145,19 @@ public void SaveMatrixAsCSVAsThirdNormalizedAndLoad() FirstIndexHeader = Helper.CreateParameter("First"), SecondIndexHeader = Helper.CreateParameter("Second"), DataIndexHeader = Helper.CreateParameter("Data"), - ThirdNormalized = Helper.CreateParameter(true) + ThirdNormalized = Helper.CreateParameter(true), + Compressed = Helper.CreateParameter(true) }; var categories = MapHelper.LoadMap(MapHelper.WriteCSV(2000)); var a = new Matrix(categories, categories); + module.Matrix = Helper.CreateParameter(a); var fileName = Path.GetTempFileName(); try { using (var writeStream = Helper.CreateWriteStreamFromFile(fileName)) { - module.Invoke((a, writeStream)); + module.WriteTo = Helper.CreateParameter(writeStream); + module.Invoke(); } // Now that the matrix has been saved attempt to load it back in. using (var readStream = Helper.CreateReadStreamFromFile(fileName)) @@ -155,6 +169,7 @@ public void SaveMatrixAsCSVAsThirdNormalizedAndLoad() OriginColumn = Helper.CreateParameter(0), DestinationColumn = Helper.CreateParameter(1), DataColumn = Helper.CreateParameter(2), + Compressed = Helper.CreateParameter(true) }.Invoke(readStream); string? error = null; Assert.IsTrue(MatrixHelper.Compare(a, readMatrix, ref error), error); diff --git a/tests/TMG-Framework.Test/Saving/TestSaveMatrixAsMTX.cs b/tests/TMG-Framework.Test/Saving/TestSaveMatrixAsMTX.cs index 97db16e..80e6ec5 100644 --- a/tests/TMG-Framework.Test/Saving/TestSaveMatrixAsMTX.cs +++ b/tests/TMG-Framework.Test/Saving/TestSaveMatrixAsMTX.cs @@ -36,7 +36,12 @@ public void SaveMatrixAsMTXAsMatrix() { using (var writeStream = Helper.CreateWriteStreamFromFile(fileName)) { - new SaveMatrixAsMTX().Invoke((a, writeStream)); + new SaveMatrixAsMTX() + { + Matrix = Helper.CreateParameter(a), + WriteTo = Helper.CreateParameter(writeStream), + Compressed = Helper.CreateParameter(false) + }.Invoke(); } } finally @@ -59,19 +64,25 @@ public void SaveMatrixAsMTXAsMatrixAndLoad() { using (var writeStream = Helper.CreateWriteStreamFromFile(fileName)) { - new SaveMatrixAsMTX().Invoke((a, writeStream)); + new SaveMatrixAsMTX() + { + Matrix = Helper.CreateParameter(a), + WriteTo = Helper.CreateParameter(writeStream), + Compressed = Helper.CreateParameter(true) + }.Invoke(); } // Now that the matrix has been saved attempt to load it back in. - using (var readStream = Helper.CreateReadStreamFromFile(fileName)) + using var readStream = Helper.CreateReadStreamFromFile(fileName); + + var readMatrix = new TMG.Loading.LoadMatrixFromMTX() { - var readMatrix = new TMG.Loading.LoadMatrixFromMTX() - { - Categories = Helper.CreateParameter(categories), - ConvertBetweenZoneSystems = Helper.CreateParameter(false) - }.Invoke(readStream); - string? error = null; - Assert.IsTrue(MatrixHelper.Compare(a, readMatrix, ref error), error); - } + Input = Helper.CreateParameter(readStream), + Categories = Helper.CreateParameter(categories), + ConvertBetweenZoneSystems = Helper.CreateParameter(false), + Compressed = Helper.CreateParameter(true) + }.Invoke(); + string? error = null; + Assert.IsTrue(MatrixHelper.Compare(a, readMatrix, ref error), error); } finally { diff --git a/tests/TMG-Framework.Test/Saving/TestSaveVectorAsCSV.cs b/tests/TMG-Framework.Test/Saving/TestSaveVectorAsCSV.cs index b3765a4..76f8f6e 100644 --- a/tests/TMG-Framework.Test/Saving/TestSaveVectorAsCSV.cs +++ b/tests/TMG-Framework.Test/Saving/TestSaveVectorAsCSV.cs @@ -38,7 +38,8 @@ public void SaveVectorAsCSV() new SaveVectorAsCSV() { MapColumnName = Helper.CreateParameter("Zones"), - DataColumnName = Helper.CreateParameter("Data") + DataColumnName = Helper.CreateParameter("Data"), + Compressed = Helper.CreateParameter(false) }.Invoke((a, writeStream)); } } @@ -65,7 +66,8 @@ public void SaveVectorAsCSVAndLoad() new SaveVectorAsCSV() { MapColumnName = Helper.CreateParameter("Zones"), - DataColumnName = Helper.CreateParameter("Data") + DataColumnName = Helper.CreateParameter("Data"), + Compressed = Helper.CreateParameter(true) }.Invoke((a, writeStream)); } // Now that the matrix has been saved attempt to load it back in. @@ -75,7 +77,8 @@ public void SaveVectorAsCSVAndLoad() { Categories = Helper.CreateParameter(categories), MapColumn = Helper.CreateParameter(0), - DataColumn = Helper.CreateParameter(1) + DataColumn = Helper.CreateParameter(1), + Compressed = Helper.CreateParameter(true) }.Invoke(readStream); string? error = null; Assert.IsTrue(Compare(a, readVector, ref error), error); diff --git a/tests/TMG-Framework.Test/Utilities/MatrixHelper.cs b/tests/TMG-Framework.Test/Utilities/MatrixHelper.cs index 887629f..1a0f2af 100644 --- a/tests/TMG-Framework.Test/Utilities/MatrixHelper.cs +++ b/tests/TMG-Framework.Test/Utilities/MatrixHelper.cs @@ -37,13 +37,16 @@ internal static string WriteMatrixToCSVMatrix(Categories categories, float[][] d } var save = new SaveMatrixAsCSV() { + Matrix = Helper.CreateParameter(matrix), ThirdNormalized = Helper.CreateParameter(false), + Compressed = Helper.CreateParameter(false), FirstIndexHeader = Helper.CreateParameter("Origin"), SecondIndexHeader = Helper.CreateParameter("Destination") }; using (var writeStream = Helper.CreateWriteStreamFromFile(fileName)) { - save.Invoke((matrix, writeStream)); + save.WriteTo = Helper.CreateParameter(writeStream); + save.Invoke(); } return fileName; } @@ -68,14 +71,17 @@ internal static string WriteMatrixToCSVThirdNormalized(Categories categories, fl } var save = new SaveMatrixAsCSV() { + Matrix = Helper.CreateParameter(matrix), ThirdNormalized = Helper.CreateParameter(true), + Compressed = Helper.CreateParameter(false), FirstIndexHeader = Helper.CreateParameter("Origin"), SecondIndexHeader = Helper.CreateParameter("Destination"), DataIndexHeader = Helper.CreateParameter("Data") }; using (var writeStream = Helper.CreateWriteStreamFromFile(fileName)) { - save.Invoke((matrix, writeStream)); + save.WriteTo = Helper.CreateParameter(writeStream); + save.Invoke(); } return fileName; } @@ -125,10 +131,15 @@ internal static string WriteMatrixToMTX(Categories categories, float[][] data) { data[i].AsSpan().CopyTo(matrix.GetRow(i)); } - var save = new SaveMatrixAsMTX(); + var save = new SaveMatrixAsMTX() + { + Matrix = Helper.CreateParameter(matrix), + Compressed = Helper.CreateParameter(false) + }; using (var writeStream = Helper.CreateWriteStreamFromFile(fileName)) { - save.Invoke((matrix, writeStream)); + save.WriteTo = Helper.CreateParameter(writeStream); + save.Invoke(); } return fileName; }