Add sourcemap support

Author: fdintino

pipeline/compilers/coffee.py

@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import os
+
 from pipeline.conf import settings
 from pipeline.compilers import SubProcessCompiler
 
@@ -13,10 +15,16 @@ def match_file(self, path):
     def compile_file(self, infile, outfile, outdated=False, force=False):
         if not outdated and not force:
             return  # File doesn't need to be recompiled
+
+        args = list(settings.COFFEE_SCRIPT_ARGUMENTS)
+        if settings.OUTPUT_SOURCEMAPS and not(set(args) & set(['-m', '--map'])):
+            args.append('--map')
+
         command = (
             settings.COFFEE_SCRIPT_BINARY,
-            "-cp",
-            settings.COFFEE_SCRIPT_ARGUMENTS,
+            "-c",
+            "-o", os.path.dirname(outfile),
+            args,
             infile,
         )
-        return self.execute_command(command, stdout_captured=outfile)
+        return self.execute_command(command, cwd=os.path.dirname(outfile))

pipeline/compilers/es6.py

@@ -13,9 +13,16 @@ def match_file(self, path):
     def compile_file(self, infile, outfile, outdated=False, force=False):
         if not outdated and not force:
             return  # File doesn't need to be recompiled
+
+        args = list(settings.BABEL_ARGUMENTS)
+
+        sourcemap_flags = set(['-s', '--source-maps'])
+        if settings.OUTPUT_SOURCEMAPS and not(set(args) & sourcemap_flags):
+            args += ['--source-maps', 'true']
+
         command = (
             settings.BABEL_BINARY,
-            settings.BABEL_ARGUMENTS,
+            args,
             infile,
             "-o",
             outfile

pipeline/compilers/less.py

@@ -14,9 +14,15 @@ def match_file(self, filename):
 
     def compile_file(self, infile, outfile, outdated=False, force=False):
         # Pipe to file rather than provide outfile arg due to a bug in lessc
+        args = list(settings.LESS_ARGUMENTS)
+
+        if settings.OUTPUT_SOURCEMAPS and '--source-map' not in args:
+            args += ['--source-map']
+
         command = (
             settings.LESS_BINARY,
-            settings.LESS_ARGUMENTS,
+            args,
             infile,
+            outfile,
         )
-        return self.execute_command(command, cwd=dirname(infile), stdout_captured=outfile)
+        return self.execute_command(command, cwd=dirname(infile))

pipeline/compilers/livescript.py

@@ -1,5 +1,8 @@
 from __future__ import unicode_literals
 
+from os.path import dirname, basename
+import json
+
 from pipeline.conf import settings
 from pipeline.compilers import SubProcessCompiler
 
@@ -13,10 +16,25 @@ def match_file(self, path):
     def compile_file(self, infile, outfile, outdated=False, force=False):
         if not outdated and not force:
             return  # File doesn't need to be recompiled
+
+        args = list(settings.LIVE_SCRIPT_ARGUMENTS)
+        if settings.OUTPUT_SOURCEMAPS and not(set(args) & set(['-m', '--map'])):
+            args += ['--map', 'linked']
+
         command = (
             settings.LIVE_SCRIPT_BINARY,
-            "-cp",
-            settings.LIVE_SCRIPT_ARGUMENTS,
+            "-c",
+            "-o", dirname(outfile),
+            args,
             infile,
         )
-        return self.execute_command(command, stdout_captured=outfile)
+        ret = self.execute_command(command, cwd=dirname(outfile))
+
+        if settings.OUTPUT_SOURCEMAPS:
+            with open("%s.map" % outfile) as f:
+                source_map = json.loads(f.read())
+            source_map['sources'] = map(basename, source_map['sources'])
+            with open("%s.map" % outfile, mode='w') as f:
+                f.write(json.dumps(source_map))
+
+        return ret

pipeline/compilers/sass.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import re
 from os.path import dirname
 
 from pipeline.conf import settings
@@ -9,14 +10,36 @@
 class SASSCompiler(SubProcessCompiler):
     output_extension = 'css'
 
+    _sass_types = {}
+
+    @property
+    def sass_type(self):
+        bin = " ".join(settings.SASS_BINARY)
+        if bin not in self._sass_types:
+            if re.search(r'node\-sass', bin):
+                self._sass_types[bin] = 'node'
+            elif re.search(r'sassc', bin):
+                self._sass_types[bin] = 'libsass'
+            else:
+                self._sass_types[bin] = 'ruby'
+        return self._sass_types[bin]
+
     def match_file(self, filename):
         return filename.endswith(('.scss', '.sass'))
 
     def compile_file(self, infile, outfile, outdated=False, force=False):
-        command = (
-            settings.SASS_BINARY,
-            settings.SASS_ARGUMENTS,
-            infile,
-            outfile
-        )
+        args = list(settings.SASS_ARGUMENTS)
+
+        if settings.OUTPUT_SOURCEMAPS:
+            if self.sass_type == 'node':
+                if '--source-map' not in args:
+                    args += ['--source-map', 'true']
+            elif self.sass_type == 'libsass':
+                if not(set(args) & set(['-m', 'g', '--sourcemap'])):
+                    args += ['--sourcemap']
+            else:
+                if not any([re.search(r'^\-\-sourcemap', a) for a in args]):
+                    args += ['--sourcemap=auto']
+
+        command = (settings.SASS_BINARY, args, infile, outfile)
         return self.execute_command(command, cwd=dirname(infile))

pipeline/compilers/stylus.py

@@ -13,9 +13,15 @@ def match_file(self, filename):
         return filename.endswith('.styl')
 
     def compile_file(self, infile, outfile, outdated=False, force=False):
+        args = list(settings.STYLUS_ARGUMENTS)
+
+        sourcemap_flags = set(['-s', '--sourcemap'])
+        if settings.OUTPUT_SOURCEMAPS and not(set(args) & sourcemap_flags):
+            args += ['--sourcemap']
+
         command = (
             settings.STYLUS_BINARY,
-            settings.STYLUS_ARGUMENTS,
+            args,
             infile
         )
         return self.execute_command(command, cwd=dirname(infile))

pipeline/compressors/__init__.py

@@ -5,6 +5,7 @@
 import posixpath
 import re
 import subprocess
+import warnings
 
 from itertools import takewhile
 
@@ -57,29 +58,46 @@ def css_compressor(self):
 
     def compress_js(self, paths, templates=None, **kwargs):
         """Concatenate and compress JS files"""
+        compressor = self.js_compressor
+
+        if settings.OUTPUT_SOURCEMAPS:
+            if hasattr(compressor, 'compress_js_with_source_map'):
+                if templates:
+                    warnings.warn("Source maps are not supported with javascript templates")
+                else:
+                    return compressor(verbose=self.verbose).compress_js_with_source_map(paths)
+
         js = self.concatenate(paths)
         if templates:
             js = js + self.compile_templates(templates)
 
         if not settings.DISABLE_WRAPPER:
             js = "(function() {\n%s\n}).call(this);" % js
 
-        compressor = self.js_compressor
         if compressor:
             js = getattr(compressor(verbose=self.verbose), 'compress_js')(js)
 
-        return js
+        return js, None
 
     def compress_css(self, paths, output_filename, variant=None, **kwargs):
         """Concatenate and compress CSS files"""
-        css = self.concatenate_and_rewrite(paths, output_filename, variant)
         compressor = self.css_compressor
+
+        if settings.OUTPUT_SOURCEMAPS:
+            if hasattr(compressor, 'compress_css_with_source_map'):
+                if variant == "datauri":
+                    warnings.warn("Source maps are not supported with datauri variant")
+                else:
+                    return (compressor(verbose=self.verbose)
+                                .compress_css_with_source_map(paths, output_filename))
+
+        css = self.concatenate_and_rewrite(paths, output_filename, variant)
         if compressor:
             css = getattr(compressor(verbose=self.verbose), 'compress_css')(css)
         if not variant:
-            return css
+            return css, None
         elif variant == "datauri":
-            return self.with_data_uri(css)
+            return self.with_data_uri(css), None
         else:
             raise CompressorError("\"%s\" is not a valid variant" % variant)
 
@@ -235,16 +253,17 @@ def filter_js(self, js):
 
 
 class SubProcessCompressor(CompressorBase):
-    def execute_command(self, command, content):
+    def execute_command(self, command, content=None, **kwargs):
         argument_list = []
         for flattening_arg in command:
             if isinstance(flattening_arg, string_types):
                 argument_list.append(flattening_arg)
             else:
                 argument_list.extend(flattening_arg)
+        stdin = subprocess.PIPE if content else None
 
         pipe = subprocess.Popen(argument_list, stdout=subprocess.PIPE,
-                                stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+                                stdin=stdin, stderr=subprocess.PIPE, **kwargs)
         if content:
             content = smart_bytes(content)
         stdout, stderr = pipe.communicate(content)

pipeline/compressors/cleancss.py

@@ -1,11 +1,66 @@
 from __future__ import unicode_literals
 
+import codecs
+import json
+import os
+
+from django.contrib.staticfiles.storage import staticfiles_storage
+
 from pipeline.conf import settings
 from pipeline.compressors import SubProcessCompressor
+from pipeline.utils import source_map_re, relurl
 
 
 class CleanCSSCompressor(SubProcessCompressor):
 
     def compress_css(self, css):
         args = [settings.CLEANCSS_BINARY, settings.CLEANCSS_ARGUMENTS]
         return self.execute_command(args, css)
+
+    def compress_css_with_source_map(self, paths, output_filename):
+        output_path = staticfiles_storage.path(output_filename)
+        output_dir = os.path.dirname(output_path)
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        args = [settings.CLEANCSS_BINARY]
+        args += ['--source-map']
+        if settings.CLEANCSS_ARGUMENTS:
+            args += [settings.CLEANCSS_ARGUMENTS]
+        else:
+            # At present, without these arguments, cleancss does not
+            # generate accurate source maps
+            args += [
+                '--skip-advanced', '--skip-media-merging',
+                '--skip-restructuring', '--skip-shorthand-compacting',
+                '--keep-line-breaks']
+        args += ['--output', output_path]
+        args += [staticfiles_storage.path(p) for p in paths]
+
+        self.execute_command(args, cwd=output_dir)
+
+        source_map_file = "%s.map" % output_path
+
+        with codecs.open(output_path, encoding='utf-8') as f:
+            css = f.read()
+        with codecs.open(source_map_file, encoding='utf-8') as f:
+            source_map = f.read()
+
+        # Strip out existing source map comment (it will be re-added with packaging)
+        css = source_map_re.sub('', css)
+
+        output_url = "%s/%s" % (
+            staticfiles_storage.url(os.path.dirname(output_filename)),
+            os.path.basename(output_path))
+
+        # Grab urls from staticfiles storage (in case filenames are hashed)
+        source_map_data = json.loads(source_map)
+        for i, source in enumerate(source_map_data['sources']):
+            source_abs_path = os.path.join(output_dir, source)
+            source_rel_path = os.path.relpath(
+                source_abs_path, staticfiles_storage.base_location)
+            source_url = staticfiles_storage.url(source_rel_path)
+            source_map_data['sources'][i] = relurl(source_url, output_url)
+        source_map = json.dumps(source_map_data)
+
+        return css, source_map

pipeline/compressors/closure.py

@@ -1,10 +1,56 @@
 from __future__ import unicode_literals
 
+import os
+import tempfile
+
+from django.contrib.staticfiles.storage import staticfiles_storage
+
 from pipeline.conf import settings
 from pipeline.compressors import SubProcessCompressor
+from pipeline.utils import source_map_re
 
 
 class ClosureCompressor(SubProcessCompressor):
+
     def compress_js(self, js):
         command = (settings.CLOSURE_BINARY, settings.CLOSURE_ARGUMENTS)
         return self.execute_command(command, js)
+
+    def compress_js_with_source_map(self, paths):
+        args = [settings.CLOSURE_BINARY, settings.CLOSURE_ARGUMENTS]
+
+        location_maps = set([])
+
+        abs_paths = []
+        for path in paths:
+            abs_path = staticfiles_storage.path(path)
+            location_maps.add("%s|%s" % (
+                os.path.dirname(abs_path),
+                staticfiles_storage.url(os.path.dirname(path))))
+            abs_paths.append(abs_path)
+            with open(abs_path) as f:
+                content = f.read()
+            matches = source_map_re.search(content)
+            if matches:
+                input_source_map = filter(None, matches.groups())[0]
+                input_source_map_file = os.path.join(os.path.dirname(abs_path), input_source_map)
+                args += [
+                    '--source_map_input',
+                    "%s|%s" %  (abs_path, input_source_map_file)]
+        for location_map in location_maps:
+            args += ['--source_map_location_mapping', location_map]
+
+        temp_file = tempfile.NamedTemporaryFile()
+
+        args += ["--create_source_map", temp_file.name]
+        for path in abs_paths:
+            args += ["--js", path]
+
+        js = self.execute_command(args, None)
+
+        with open(temp_file.name) as f:
+            source_map = f.read()
+
+        temp_file.close()
+
+        return js, source_map