Support saving and loading txt files, fix minor bugs in error reporting

Author: Ludwig Schubert

lucid/misc/io/loading.py

@@ -73,9 +73,15 @@ def _load_json(handle, **kwargs):
     return json.load(handle)
 
 
-def _load_text(handle, encoding="utf-8"):
+def _load_text(handle, split=False, encoding="utf-8"):
     """Load and decode a string."""
-    return handle.read().decode(encoding)
+
+    string = handle.read().decode(encoding)
+
+    if split:
+        return string.splitlines()
+    else:
+        return string
 
 
 def _load_graphdef_protobuf(handle, **kwargs):

lucid/misc/io/reading.py

@@ -110,8 +110,6 @@ def read_handle(url, cache=None, mode="rb"):
     else:
         if scheme in ("http", "https"):
             handle = _handle_web_url(url, mode=mode)
-        elif scheme == "gs":
-            handle = _handle_gcs_url(url, mode=mode)
         else:
             handle = _handle_gfile(url, mode=mode)
 
@@ -130,14 +128,6 @@ def _handle_web_url(url, mode="r"):
     return request.urlopen(url)
 
 
-def _handle_gcs_url(url, mode="r"):
-    # TODO: transparently allow authenticated access through storage API
-    _, resource_name = url.split("://")
-    base_url = "https://storage.googleapis.com/"
-    url = urljoin(base_url, resource_name)
-    return _handle_web_url(url, mode=mode)
-
-
 # Helper Functions
 
 

lucid/misc/io/saving.py

@@ -16,23 +16,26 @@
 """Method for saving arbitrary data to arbitrary destinations.
 
 This module takes an object and URL, infers how to serialize and how to write
-it out to the destination.
+it out to the destination. The intention is to preserve your work under most
+circumstances, so sometimes this will convert values by default and warn rather than
+error out immediately. This sometimes means less predictable behavior.
 
-If an object could have multiple representations, this tries to infer the
-intended representation from the URL's file extension.
+If an object could have multiple serializations, this tries to infer the
+intended serializations from the URL's file extension.
 
 Possible extension: if not given a URL this could create one and return it?
 """
 
 from __future__ import absolute_import, division, print_function
 
 import logging
+import warnings
 import os.path
 import json
 import numpy as np
 import PIL.Image
 
-from lucid.misc.io.writing import write, write_handle
+from lucid.misc.io.writing import write_handle
 from lucid.misc.io.serialize_array import _normalize_array
 
 
@@ -41,82 +44,100 @@
 
 
 def save_json(object, handle, indent=2):
-  """Save object as json on CNS."""
-  obj_json = json.dumps(object, indent=indent)
-  handle.write(obj_json)
+    """Save object as json on CNS."""
+    obj_json = json.dumps(object, indent=indent)
+    handle.write(obj_json)
 
 
 def save_npy(object, handle):
-  """Save numpy array as npy file."""
-  np.save(handle, object)
+    """Save numpy array as npy file."""
+    np.save(handle, object)
 
 
 def save_npz(object, handle):
-  """Save dict of numpy array as npz file."""
-  # there is a bug where savez doesn't actually accept a file handle.
-  log.warning("Saving npz files currently only works locally. :/")
-  path = handle.name
-  handle.close()
-  if type(object) is dict:
-    np.savez(path, **object)
-  elif type(object) is list:
-    np.savez(path, *object)
-  else:
-    log.warning("Saving non dict or list as npz file, did you maybe want npy?")
-    np.savez(path, object)
+    """Save dict of numpy array as npz file."""
+    # there is a bug where savez doesn't actually accept a file handle.
+    log.warning("Saving npz files currently only works locally. :/")
+    path = handle.name
+    handle.close()
+    if type(object) is dict:
+        np.savez(path, **object)
+    elif type(object) is list:
+        np.savez(path, *object)
+    else:
+        log.warning("Saving non dict or list as npz file, did you maybe want npy?")
+        np.savez(path, object)
 
 
 def save_img(object, handle, **kwargs):
-  """Save numpy array as image file on CNS."""
-  if isinstance(object, np.ndarray):
-    normalized = _normalize_array(object)
-    image = PIL.Image.fromarray(normalized)
-  elif isinstance(object, PIL.Image):
-    image = object
-  else:
-    raise ValueError("Can only save_img for numpy arrays or PIL.Images!")
-  image.save(handle, **kwargs)  # will infer format from handle's url ext.
+    """Save numpy array as image file on CNS."""
+
+    if isinstance(object, np.ndarray):
+        normalized = _normalize_array(object)
+        object = PIL.Image.fromarray(normalized)
+
+    if isinstance(object, PIL.Image.Image):
+        object.save(handle, **kwargs)  # will infer format from handle's url ext.
+    else:
+        raise ValueError("Can only save_img for numpy arrays or PIL.Images!")
+
+
+def save_txt(object, handle, **kwargs):
+    if isinstance(object, str):
+        handle.write(object)
+    elif isinstance(object, list):
+        for line in object:
+            if not isinstance(line, str):
+                line = repr(line)
+                warnings.warn(
+                    "`save_txt` was called with an array containing objects other than "
+                    "strings; using `repr` to convert values to string."
+                )
+            if not line.endswith("\n"):
+                line += "\n"
+            handle.write(line)
 
 
 savers = {
-  ".png": save_img,
-  ".jpg": save_img,
-  ".jpeg": save_img,
-  ".npy": save_npy,
-  ".npz": save_npz,
-  ".json": save_json,
+    ".png": save_img,
+    ".jpg": save_img,
+    ".jpeg": save_img,
+    ".npy": save_npy,
+    ".npz": save_npz,
+    ".json": save_json,
+    ".txt": save_txt,
 }
 
 
 def save(thing, url_or_handle, **kwargs):
-  """Save object to file on CNS.
-
-  File format is inferred from path. Use save_img(), save_npy(), or save_json()
-  if you need to force a particular format.
-
-  Args:
-    obj: object to save.
-    path: CNS path.
-
-  Raises:
-    RuntimeError: If file extension not supported.
-  """
-  is_handle = hasattr(url_or_handle, 'write') and hasattr(url_or_handle, 'name')
-  if is_handle:
-    _, ext = os.path.splitext(url_or_handle.name)
-  else:
-    _, ext = os.path.splitext(url_or_handle)
-  if not ext:
-    raise RuntimeError("No extension in URL: " + url_or_handle)
-
-  if ext in savers:
-    saver = savers[ext]
+    """Save object to file on CNS.
+
+    File format is inferred from path. Use save_img(), save_npy(), or save_json()
+    if you need to force a particular format.
+
+    Args:
+      obj: object to save.
+      path: CNS path.
+
+    Raises:
+      RuntimeError: If file extension not supported.
+    """
+    is_handle = hasattr(url_or_handle, "write") and hasattr(url_or_handle, "name")
     if is_handle:
-      saver(thing, url_or_handle, **kwargs)
+        _, ext = os.path.splitext(url_or_handle.name)
+    else:
+        _, ext = os.path.splitext(url_or_handle)
+    if not ext:
+        raise RuntimeError("No extension in URL: " + url_or_handle)
+
+    if ext in savers:
+        saver = savers[ext]
+        if is_handle:
+            saver(thing, url_or_handle, **kwargs)
+        else:
+            with write_handle(url_or_handle) as handle:
+                saver(thing, handle, **kwargs)
     else:
-      with write_handle(url_or_handle) as handle:
-        saver(thing, handle, **kwargs)
-  else:
-    saver_names = [(key, function.__name__) for (key, value) in savers.items()]
-    message = "Unknown extension '{}', supports {}."
-    raise RuntimeError(message.format(ext, saver_names))
+        saver_names = [(key, fn.__name__) for (key, fn) in savers.items()]
+        message = "Unknown extension '{}', supports {}."
+        raise ValueError(message.format(ext, saver_names))

tests/fixtures/multiline.txt

@@ -0,0 +1,10 @@
+Line 0
+Line 1
+Line 2
+Line 3
+Line 4
+Line 5
+Line 6
+Line 7
+Line 8
+Line 9

tests/misc/io/test_loading.py

@@ -21,6 +21,13 @@ def test_load_text():
   assert u"🐕" in string
 
 
+def test_load_multiline_text_as_list():
+  path = "./tests/fixtures/multiline.txt"
+  string_list = load(path, split=True)
+  assert isinstance(string_list, list)
+  assert all(isinstance(string, str) for string in string_list)
+
+
 def test_load_npy():
   path = "./tests/fixtures/array.npy"
   array = load(path)