Add read_file and write_file functions to work with text files taking settings.FILE_CHARSET into consideration

Author: Andrey Fedoseev

static_precompiler/compilers/base.py

@@ -158,8 +158,7 @@ def get_source(self, source_path):
         :returns: str
 
         """
-        with open(self.get_full_source_path(source_path)) as source:
-            return source.read()
+        return utils.read_file(self.get_full_source_path(source_path))
 
     def compile(self, source_path, from_management=False, verbosity=0):
         """ Compile the given source path and return relative path to the compiled file.

static_precompiler/compilers/libsass.py

@@ -64,15 +64,12 @@ def compile_file(self, source_path):
         compiled = encoding.force_str(compiled)
         sourcemap = encoding.force_str(sourcemap)
 
-        with open(full_output_path, "w+") as output_file:
-            output_file.write(compiled)
+        utils.write_file(compiled, full_output_path)
 
         utils.convert_urls(full_output_path, source_path)
 
         if self.is_sourcemap_enabled:
-            with open(sourcemap_path, "w+") as output_file:
-                output_file.write(sourcemap)
-
+            utils.write_file(sourcemap, sourcemap_path)
             utils.fix_sourcemap(sourcemap_path, source_path, full_output_path)
 
         return self.get_output_path(source_path)

static_precompiler/tests/test_utils.py

@@ -1,8 +1,23 @@
+# coding: utf-8
+from __future__ import unicode_literals
 import django.core.exceptions
 import pytest
 from pretend import stub
 
 from static_precompiler import compilers, exceptions, utils
+import os
+import six
+
+
+def test_write_read_file(tmpdir, settings):
+    settings.FILE_CHARSET = "utf-8"
+    path = os.path.join(tmpdir.dirname, "foo.txt")
+    utils.write_file("Привет, Мир!", path)
+
+    assert os.path.exists(path)
+    read_content = utils.read_file(path)
+    assert isinstance(read_content, six.text_type)
+    assert read_content == "Привет, Мир!"
 
 
 def test_build_compilers(monkeypatch):

static_precompiler/utils.py

@@ -10,6 +10,7 @@
 
 import django.core.cache
 import django.core.exceptions
+import django.conf
 from django.utils import encoding, six
 
 from static_precompiler import exceptions, settings
@@ -21,10 +22,10 @@
 
 
 if six.PY2:
-    # noinspection PyUnresolvedReferences
+    # noinspection PyUnresolvedReferences,PyCompatibility
     from urlparse import urljoin
 else:
-    # noinspection PyUnresolvedReferences
+    # noinspection PyUnresolvedReferences,PyCompatibility
     from urllib.parse import urljoin
 
 
@@ -36,6 +37,30 @@ def normalize_path(posix_path):
     return os.path.join(*posix_path.split("/"))
 
 
+def read_file(path):
+    """ Return the contents of a file as unicode. """
+    if six.PY2:
+        with open(path) as file_object:
+            return file_object.read().decode(django.conf.settings.FILE_CHARSET)
+    else:
+        with open(path, encoding=django.conf.settings.FILE_CHARSET) as file_object:
+            return file_object.read()
+
+
+def write_file(content, path):
+    """ Write unicode content to a file. """
+
+    # Convert to unicode
+    content = encoding.force_text(content)
+
+    if six.PY2:
+        with open(path, "w+") as file_object:
+            file_object.write(content.encode(django.conf.settings.FILE_CHARSET))
+    else:
+        with open(path, "w+", encoding=django.conf.settings.FILE_CHARSET) as file_object:
+            file_object.write(content)
+
+
 def fix_line_breaks(text):
     """ Convert Win line breaks to Unix
     """
@@ -131,10 +156,9 @@ def convert(self, content, path):
 
 
 def convert_urls(compiled_full_path, source_path):
-    with open(compiled_full_path, "r+") as compiled_file:
-        content = compiled_file.read()
-    with open(compiled_full_path, "w") as compiled_file:
-        compiled_file.write(url_converter.convert(content, source_path))
+    content = read_file(compiled_full_path)
+    converted_content = url_converter.convert(content, source_path)
+    write_file(converted_content, compiled_full_path)
 
 
 def build_compilers():
@@ -218,14 +242,12 @@ def compile_static_lazy(path):
 
 def fix_sourcemap(sourcemap_full_path, source_path, compiled_full_path):
 
-    with open(sourcemap_full_path) as sourcemap_file:
-        sourcemap = json.loads(sourcemap_file.read())
+    sourcemap = json.loads(read_file(sourcemap_full_path))
 
     # Stylus, unlike SASS, can't add correct relative paths in source map when the compiled file
     # is not in the same dir as the source file. We fix it here.
     sourcemap["sourceRoot"] = "../" * len(source_path.split("/")) + posixpath.dirname(source_path)
     sourcemap["sources"] = [os.path.basename(source) for source in sourcemap["sources"]]
     sourcemap["file"] = posixpath.basename(os.path.basename(compiled_full_path))
 
-    with open(sourcemap_full_path, "w") as sourcemap_file:
-        sourcemap_file.write(json.dumps(sourcemap))
+    write_file(json.dumps(sourcemap), sourcemap_full_path)