Fix escaped quotes

src/template_compiler_scanner.erl

@@ -3,9 +3,9 @@
 %%% @author    Evan Miller <[email protected]>
 %%% @author    Marc Worrell <[email protected]>
 %%% @copyright 2008 Roberto Saccon, Evan Miller; 2009-2020 Marc Worrell
-%%% @doc 
+%%% @doc
 %%% Template language scanner
-%%% @end  
+%%% @end
 %%%
 %%% The MIT License
 %%%
@@ -43,7 +43,7 @@
 -author('[email protected]').
 -author('[email protected]').
 
--export([scan/1, scan/2]). 
+-export([scan/1, scan/2]).
 
 -define(IS_EOF(S),
         (S =:= <<>> orelse S =:= <<"\n">> orelse S =:= <<"\r\n">>)).
@@ -68,11 +68,11 @@ scan(SourceRef, Template) when is_binary(Template) ->
     scan(Template, [], {SourceRef, 1, 1}, in_text).
 
 
-identifier_to_keyword({identifier, Pos, String}, {PrevToken, Acc}) 
-  when PrevToken =:= open_tag; 
+identifier_to_keyword({identifier, Pos, String}, {PrevToken, Acc})
+  when PrevToken =:= open_tag;
        PrevToken =:= all_keyword;
        PrevToken =:= optional_keyword ->
-    %% the last two guards really ought to be for it's own fun, 
+    %% the last two guards really ought to be for it's own fun,
     %% since they only apply for [cat]include (the last one only for include)
 
     %% At the start of a {% .... %} tag we accept all keywords
@@ -93,12 +93,12 @@ identifier_to_keyword({identifier, Pos, String}, {PrevToken, Acc})
         <<"m">>
     ],
     Type = case lists:member(String, Keywords) of
-        true -> 
+        true ->
             case list_to_atom(binary_to_list(String) ++ "_keyword") of
                 elseif_keyword -> elif_keyword;
                 KWA -> KWA
             end;
-        _ -> 
+        _ ->
             identifier
     end,
     {Type, [{Type, Pos, String}|Acc]};
@@ -117,7 +117,7 @@ identifier_to_keyword({identifier, Pos, String}, {PrevToken, Acc}) when PrevToke
     Keywords = [
         <<"in">>, <<"not">>, <<"or">>, <<"and">>, <<"xor">>, <<"firstof">>,
         <<"regroup">>, <<"templatetag">>, <<"with">>, <<"as">>
-    ], 
+    ],
     Type = case lists:member(String, Keywords) of
         true -> list_to_atom(binary_to_list(String) ++ "_keyword");
         _ ->    identifier
@@ -130,15 +130,15 @@ identifier_to_keyword({identifier, Pos, String}, {_PrevToken, Acc}) ->
         <<"in">>, <<"not">>, <<"or">>, <<"and">>, <<"xor">>, <<"firstof">>,
         <<"regroup">>, <<"templatetag">>, <<"with">>, <<"as">>,
         <<"m">>
-    ], 
+    ],
     Type = case lists:member(String, Keywords) of
         true -> list_to_atom(binary_to_list(String) ++ "_keyword");
         _ ->    identifier
     end,
     {Type, [{Type, Pos, String}|Acc]};
 identifier_to_keyword({Type, Pos, String}, {_PrevToken, Acc}) ->
     {Type, [{Type, Pos, String}|Acc]}.
-    
+
 
 scan(Eof, Scanned, _, in_text) when ?IS_EOF(Eof) ->
     {_Token, ScannedKeyword} = lists:foldr(fun identifier_to_keyword/2, {'$eof', []}, Scanned),
@@ -200,13 +200,13 @@ scan(<<"{{", T/binary>>, Scanned, {SourceRef, Row, Column}, in_text) ->
 scan(<<"<!--{_", T/binary>>, Scanned, {SourceRef, Row, Column}, in_text) ->
     scan(T, [
             {trans_text, {SourceRef, Row, Column + 6}, <<"">>},
-            {open_trans, {SourceRef, Row, Column}, <<"<!--{_">>} | Scanned], 
+            {open_trans, {SourceRef, Row, Column}, <<"<!--{_">>} | Scanned],
     {SourceRef, Row, Column + 6}, {in_trans, <<"_}-->">>});
 
 scan(<<"{_", T/binary>>, Scanned, {SourceRef, Row, Column}, in_text) ->
     scan(T, [
             {trans_text, {SourceRef, Row, Column + 2}, <<>>},
-            {open_trans, {SourceRef, Row, Column}, <<"{_">>} | Scanned], 
+            {open_trans, {SourceRef, Row, Column}, <<"{_">>} | Scanned],
         {SourceRef, Row, Column + 2}, {in_trans, <<"_}">>});
 
 scan(<<"_}-->", T/binary>>, Scanned, {SourceRef, Row, Column}, {in_trans, <<"_}-->">>}) ->
@@ -237,11 +237,11 @@ scan(<<_/utf8, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_comment, Close
     scan(T, Scanned, {SourceRef, Row, Column + 1}, {in_comment, Closer});
 
 scan(<<"<!--{%", T/binary>>, Scanned, {SourceRef, Row, Column}, in_text) ->
-    scan(T, [{open_tag, {SourceRef, Row, Column}, <<"<!--{%">>} | Scanned], 
+    scan(T, [{open_tag, {SourceRef, Row, Column}, <<"<!--{%">>} | Scanned],
         {SourceRef, Row, Column + 6}, {in_code, <<"%}-->">>});
 
 scan(<<"{%", T/binary>>, Scanned, {SourceRef, Row, Column}, in_text) ->
-    scan(T, [{open_tag, {SourceRef, Row, Column}, <<"{%">>} | Scanned], 
+    scan(T, [{open_tag, {SourceRef, Row, Column}, <<"{%">>} | Scanned],
         {SourceRef, Row, Column + 2}, {in_code, <<"%}">>});
 
 scan(<<H/utf8, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_trans, Closer}) ->
@@ -282,41 +282,30 @@ scan(<<"_\'", T/binary>>, Scanned, {SourceRef, Row, Column}, {in_code, Closer})
 scan(<<$', T/binary>>, Scanned, {SourceRef, Row, Column}, {in_identifier, Closer}) ->
     scan(T, [{string_literal, {SourceRef, Row, Column}, <<>>} | Scanned], {SourceRef, Row, Column + 1}, {in_single_quote, Closer});
 
-scan(<<"`", T/binary>>, Scanned, {SourceRef, Row, Column}, {in_code, Closer}) ->
+scan(<<$`, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_code, Closer}) ->
     scan(T, [{atom_literal, {SourceRef, Row, Column}, <<>>} | Scanned], {SourceRef, Row, Column + 1}, {in_back_quote, Closer});
 
 scan(<<$`, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_identifier, Closer}) ->
     scan(T, [{atom_literal, {SourceRef, Row, Column}, <<>>} | Scanned], {SourceRef, Row, Column + 1}, {in_back_quote, Closer});
 
 scan(<<$\\, $", T/binary>>, Scanned, {SourceRef, Row, Column}, {in_double_quote, Closer}) ->
-    scan(T, append_char(Scanned, $"), {SourceRef, Row, Column + 1}, {in_double_quote_slash, Closer});
-
-scan(<<H/utf8, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_double_quote_slash, Closer}) ->
-    scan(T, append_char(Scanned, H), {SourceRef, Row, Column + 1}, {in_double_quote, Closer});
+    scan(T, append_char(append_char(Scanned, $\\), $"), {SourceRef, Row, Column + 2}, {in_double_quote, Closer});
 
 scan(<<$\\, $', T/binary>>, Scanned, {SourceRef, Row, Column}, {in_single_quote, Closer}) ->
-    scan(T, append_char(Scanned, $'), {SourceRef, Row, Column + 1}, {in_single_quote_slash, Closer});
-
-scan(<<H/utf8, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_single_quote_slash, Closer}) ->
-    scan(T, append_char(Scanned, H), {SourceRef, Row, Column + 1}, {in_single_quote, Closer});
-
-scan(<<$\\, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_back_quote, Closer}) ->
-    scan(T, append_char(Scanned, $\\), {SourceRef, Row, Column + 1}, {in_back_quote_slash, Closer});
-
-scan(<<H/utf8, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_back_quote_slash, Closer}) ->
-    scan(T, append_char(Scanned, H), {SourceRef, Row, Column + 1}, {in_back_quote, Closer});
-
+    scan(T, append_char(append_char(Scanned, $\\), $'), {SourceRef, Row, Column + 2}, {in_single_quote, Closer});
 
+scan(<<$\\, $`, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_back_quote, Closer}) ->
+    scan(T, append_char(append_char(Scanned, $\\), $`), {SourceRef, Row, Column + 2}, {in_back_quote, Closer});
 
 % end quote
-scan(<<"\"", T/binary>>, Scanned, {SourceRef, Row, Column}, {in_double_quote, Closer}) ->
+scan(<<$", T/binary>>, Scanned, {SourceRef, Row, Column}, {in_double_quote, Closer}) ->
     scan(T, Scanned, {SourceRef, Row, Column + 1}, {in_code, Closer});
 
 % treat single quotes the same as double quotes
-scan(<<"\'", T/binary>>, Scanned, {SourceRef, Row, Column}, {in_single_quote, Closer}) ->
+scan(<<$', T/binary>>, Scanned, {SourceRef, Row, Column}, {in_single_quote, Closer}) ->
     scan(T, Scanned, {SourceRef, Row, Column + 1}, {in_code, Closer});
 
-scan(<<"`", T/binary>>, Scanned, {SourceRef, Row, Column}, {in_back_quote, Closer}) ->
+scan(<<$`, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_back_quote, Closer}) ->
     scan(T, Scanned, {SourceRef, Row, Column + 1}, {in_code, Closer});
 
 scan(<<H/utf8, T/binary>>, Scanned, {SourceRef, Row, Column}, {in_double_quote, Closer}) ->

test/template_compiler_quote_SUITE.erl

@@ -0,0 +1,75 @@
+-module(template_compiler_quote_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+
+suite() ->
+    [
+        {timetrap, {seconds, 30}}
+    ].
+
+all() ->
+    [
+        {group, basic}
+    ].
+
+groups() ->
+    [{basic, [],
+        [single_quote_test
+        ,single_quote_slash_test
+        ,double_quote_test
+        ,double_quote_slash_test
+        ,back_quote_test
+        ,back_quote_slash_test
+        ]}].
+
+init_per_suite(Config) ->
+    {ok, _} = application:ensure_all_started(template_compiler),
+    application:set_env(template_compiler, template_dir, test_data_dir(Config)),
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(basic, Config) ->
+    Config.
+
+end_per_group(basic, _Config) ->
+    ok.
+
+single_quote_test(_Config) ->
+    {ok, Bin} = template_compiler:render("quote_single.tpl", #{}, [], undefined),
+    <<"<pre>&lt;&lt;&quot;foo&quot;&gt;&gt;</pre>'bar'">> = iolist_to_binary(Bin),
+    ok.
+
+single_quote_slash_test(_Config) ->
+    {ok, Bin} = template_compiler:render("quote_single_slash.tpl", #{}, [], undefined),
+    <<"<pre>&lt;&lt;&quot;&#39;foo&#39;&quot;&gt;&gt;</pre>\\'bar\\'">> = iolist_to_binary(Bin),
+    ok.
+
+double_quote_test(_Config) ->
+    {ok, Bin} = template_compiler:render("quote_double.tpl", #{}, [], undefined),
+    <<"<pre>&lt;&lt;&quot;foo&quot;&gt;&gt;</pre>\"bar\"">> = iolist_to_binary(Bin),
+    ok.
+
+double_quote_slash_test(_Config) ->
+    {ok, Bin} = template_compiler:render("quote_double_slash.tpl", #{}, [], undefined),
+    <<"<pre>&lt;&lt;&quot;\\&quot;foo\\&quot;&quot;&gt;&gt;</pre>\\\"bar\\\"">> = iolist_to_binary(Bin),
+    ok.
+
+back_quote_test(_Config) ->
+    {ok, Bin} = template_compiler:render("quote_back.tpl", #{}, [], undefined),
+    <<"<pre>foo</pre>`bar`">> = iolist_to_binary(Bin),
+    ok.
+
+back_quote_slash_test(_Config) ->
+    {ok, Bin} = template_compiler:render("quote_back_slash.tpl", #{}, [], undefined),
+    <<"<pre>&#39;\\\\`foo\\\\`&#39;</pre>\\`bar\\`">> = iolist_to_binary(Bin),
+    ok.
+
+test_data_dir(Config) ->
+    filename:join([
+        filename:dirname(filename:dirname(?config(data_dir, Config))),
+        "test-data"]).

test/test-data/quote_back.tpl

@@ -0,0 +1 @@
+{% print `foo` %}`bar`

test/test-data/quote_back_slash.tpl

@@ -0,0 +1 @@
+{% print `\`foo\`` %}\`bar\`

test/test-data/quote_double.tpl

@@ -0,0 +1 @@
+{% print "foo" %}"bar"

test/test-data/quote_double_slash.tpl

@@ -0,0 +1 @@
+{% print "\"foo\"" %}\"bar\"

test/test-data/quote_single.tpl

@@ -0,0 +1 @@
+{% print 'foo' %}'bar'

test/test-data/quote_single_slash.tpl

@@ -0,0 +1 @@
+{% print '\'foo\'' %}\'bar\'