From 92357f479e12f8092f9c050924ed0f7bc5c5ae26 Mon Sep 17 00:00:00 2001 From: Sam Sandberg Date: Mon, 27 Mar 2017 19:00:52 -0400 Subject: [PATCH 1/4] `./npy_to_largeviz_input.py [output.txt]` --- npy_to_largeviz_input.py | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100755 npy_to_largeviz_input.py diff --git a/npy_to_largeviz_input.py b/npy_to_largeviz_input.py new file mode 100755 index 0000000..e85f7b5 --- /dev/null +++ b/npy_to_largeviz_input.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +"""Tool for converting a numpy array file `input.npy` to the type of file that +LargeViz expects as input, `input.npy.txt`. + +TODO: `.npz` support + +References: + +- https://docs.scipy.org/doc/numpy/reference/generated/numpy.savez.html +- https://docs.scipy.org/doc/numpy/reference/generated/numpy.save.html +- https://docs.scipy.org/doc/numpy/reference/generated/numpy.load.html +- https://docs.scipy.org/doc/numpy/neps/npy-format.html + +""" + +import csv +import sys + +import numpy as np + + +def npy_to_largeviz_input(filename_input, filename_output=None): + data = np.load(filename_input) + if not filename_output: + filename_output = '%s.txt' % filename_input + with open(filename_output, 'w') as fp: + writer = csv.writer(fp, delimiter=' ') + writer.writerow(data.shape) + for vector in data: + writer.writerow(vector.tolist()) + + +def main(): + filename_input = sys.argv[1] + filename_output = len(sys.argv) > 2 and sys.argv[2] + npy_to_largeviz_input(filename_input, filename_output) + + +if __name__ == '__main__': + main() From f53ac915483c69a5b7ea998cd2c2a3b35499bdeb Mon Sep 17 00:00:00 2001 From: Sam Sandberg Date: Tue, 28 Mar 2017 10:19:50 -0400 Subject: [PATCH 2/4] Added logging, renamed function to `npy_to_largeviz_txt` --- npy_to_largeviz_input.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/npy_to_largeviz_input.py b/npy_to_largeviz_input.py index e85f7b5..54dc257 100755 --- a/npy_to_largeviz_input.py +++ b/npy_to_largeviz_input.py @@ -15,15 +15,21 @@ """ import csv +import logging import sys import numpy as np -def npy_to_largeviz_input(filename_input, filename_output=None): +logger = logging.getLogger(__name__) + + +def npy_to_largeviz_txt(filename_input, filename_output=None): + logger.info("npy_to_largeviz_txt() on %s", filename_input) data = np.load(filename_input) if not filename_output: filename_output = '%s.txt' % filename_input + logger.info("Saving result to %s", filename_output) with open(filename_output, 'w') as fp: writer = csv.writer(fp, delimiter=' ') writer.writerow(data.shape) @@ -31,10 +37,22 @@ def npy_to_largeviz_input(filename_input, filename_output=None): writer.writerow(vector.tolist()) +def setup_logger(): + logger.setLevel(logging.INFO) + logger.handlers = [] + formatter = logging.Formatter('%(asctime)s [%(levelname)s]: %(message)s', + '%Y-%m-%d %H:%M:%S') + stream_handler = logging.StreamHandler() + stream_handler.setLevel(logging.INFO) + stream_handler.setFormatter(formatter) + logger.addHandler(stream_handler) + + def main(): + setup_logger() filename_input = sys.argv[1] filename_output = len(sys.argv) > 2 and sys.argv[2] - npy_to_largeviz_input(filename_input, filename_output) + npy_to_largeviz_txt(filename_input, filename_output) if __name__ == '__main__': From 488e364f3305c6406158ca2795f4e9486dead5df Mon Sep 17 00:00:00 2001 From: Sam Sandberg Date: Tue, 28 Mar 2017 10:20:08 -0400 Subject: [PATCH 3/4] Added function to go the other way, `largeviz_txt_to_npy()` --- npy_to_largeviz_input.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/npy_to_largeviz_input.py b/npy_to_largeviz_input.py index 54dc257..bb2136d 100755 --- a/npy_to_largeviz_input.py +++ b/npy_to_largeviz_input.py @@ -37,6 +37,21 @@ def npy_to_largeviz_txt(filename_input, filename_output=None): writer.writerow(vector.tolist()) +def largeviz_txt_to_npy(filename_input, filename_output=None): + logger.info("largeviz_txt_to_npy() on %s", filename_input) + with open(filename_input, 'r') as fp: + reader = csv.reader(fp, delimiter=' ') + shape = map(int, reader.next()) + logger.info("shape=%s", shape) + vectors = np.zeros(shape) + for row_num, row in enumerate(reader): + vectors[row_num, :] = np.array(row) + if not filename_output: + filename_output = '%s.npy' % filename_input + logger.info("Saving result to %s", filename_output) + np.save(filename_output, vectors) + + def setup_logger(): logger.setLevel(logging.INFO) logger.handlers = [] @@ -52,7 +67,10 @@ def main(): setup_logger() filename_input = sys.argv[1] filename_output = len(sys.argv) > 2 and sys.argv[2] - npy_to_largeviz_txt(filename_input, filename_output) + if filename_input.endswith('.txt'): + largeviz_txt_to_npy(filename_input, filename_output) + else: + npy_to_largeviz_txt(filename_input, filename_output) if __name__ == '__main__': From 2864aa849589d824eb1c2eeb86b87f4db2457c69 Mon Sep 17 00:00:00 2001 From: Sam Sandberg Date: Tue, 28 Mar 2017 10:20:56 -0400 Subject: [PATCH 4/4] Renamed file to `npy_to_largeviz_txt.py` --- npy_to_largeviz_input.py => npy_to_largeviz_txt.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename npy_to_largeviz_input.py => npy_to_largeviz_txt.py (100%) diff --git a/npy_to_largeviz_input.py b/npy_to_largeviz_txt.py similarity index 100% rename from npy_to_largeviz_input.py rename to npy_to_largeviz_txt.py