Add support for C2y-style \o and braced escapes, and \d for decimal

C2y adds \o for octal byte escapes, and allows \x, \o, or \u escape
sequences to be enclosed in braces to indicate their termination.

Add a \d sequence as a NASM extension to allow decimal bytes to be
specified.

Signed-off-by: H. Peter Anvin (Intel) <hpa@zytor.com>

Author: H. Peter Anvin (Intel)

asm/quote.c

@@ -269,54 +269,53 @@ char *nasm_quote_cstr(const char *str, size_t *lenp)
  * to indicate the lead marker of a quoted string. If it is '\"', then
  * '`' is not a special character at all.
  */
+enum unq_state {
+    st_start,
+    st_backslash,
+    st_byte,                /* Byte numeric sequence */
+    st_ucs,                 /* \u or \U */
+    st_done
+};
 
 size_t nasm_unquote_anystr(char *str, char **ep, const uint32_t badctl,
                            const char qstart)
 {
-    unsigned char bq;
-    const unsigned char *p;
-    const unsigned char *escp = NULL;
-    unsigned char *q;
+    const unsigned char bq = *str;
+    const unsigned char *p = (unsigned char *)str;
+    unsigned char *q = (unsigned char *)str;
     unsigned char c;
     uint32_t ctlmask = 0;       /* Mask of control characters seen */
-    enum unq_state {
-	st_start,
-	st_backslash,
-	st_hex,
-	st_oct,
-	st_ucs,
-        st_done
-    } state;
-    int ndig = 0;
-    uint32_t nval = 0;
-
-    p = q = (unsigned char *)str;
-
-    bq = *p++;
-    if (!bq)
-	return 0;
 
     if (bq == (unsigned char)qstart) {
-	/* `...` string */
-	state = st_start;
+	/* `...` string or "..." with C unquoting */
+        enum unq_state state = st_start;
+        unsigned int base = 0;      /* Base of numeric escape sequence */
+        uint64_t nval = 0;          /* Accumulated value of numeric sequence */
+        unsigned int v;
+        int ndig = 0;               /* Max digits of numeric sequence */
+        /* ndig < 0 means braced sequence */
+        const unsigned char *escp = NULL; /* Pointer to immediately after \ */
+
+        p++;                    /* Skip initial quote */
 
 	while (state != st_done) {
 	    c = *p++;
 	    switch (state) {
 	    case st_start:
                 if (c == '\\') {
 		    state = st_backslash;
-                } else if ((c == '\0') | (c == bq)) {
+                } else if (c == '\0' || c == bq) {
                     state = st_done;
                 } else {
                     EMIT(c);
                 }
-		break;
+                break;
 
 	    case st_backslash:
 		state = st_start;
-		escp = p;	/* Beginning of argument sequence */
+		escp = p-1;
 		nval = 0;
+
 		switch (c) {
 		case 'a':
 		    nval = 7;
@@ -341,20 +340,28 @@ size_t nasm_unquote_anystr(char *str, char **ep, const uint32_t badctl,
 		    break;
 		case 'u':
 		    state = st_ucs;
+                    base = 16;
 		    ndig = 4;
-		    break;
+                    goto check_brace;
 		case 'U':
 		    state = st_ucs;
+                    base = 16;
 		    ndig = 8;
-		    break;
+                    goto check_brace;
 		case 'v':
 		    nval = 11;
 		    break;
 		case 'x':
 		case 'X':
-		    state = st_hex;
+		    state = st_byte;
+                    base = 16;
 		    ndig = 2;
-		    break;
+                    goto check_brace;
+                case 'd':       /* NASM extension: \d = decimal */
+                    state = st_byte;
+                    base = 10;
+                    ndig = 3;
+                    goto check_brace;
 		case '0':
 		case '1':
 		case '2':
@@ -363,10 +370,19 @@ size_t nasm_unquote_anystr(char *str, char **ep, const uint32_t badctl,
 		case '5':
 		case '6':
 		case '7':
-		    state = st_oct;
-		    ndig = 2;	/* Up to two more digits */
-		    nval = c - '0';
-		    break;
+                    /* Back up both p and escp, as if there had been an "o" */
+                    p = escp--;
+                    /* fall through */
+                case 'o':
+                    state = st_byte;
+                    ndig = 3;
+                    base = 8;
+                check_brace:    /* Is this the start of a braced sequence? */
+                    if (*p == '{') {
+                        p++;    /* Skip brace */
+                        ndig = -1;
+                    }
+                    break;
                 case '\0':
                     nval = '\\';
                     p--;        /* Reprocess; terminates string */
@@ -379,38 +395,37 @@ size_t nasm_unquote_anystr(char *str, char **ep, const uint32_t badctl,
                     EMIT(nval);
 		break;
 
-	    case st_oct:
-		if (c >= '0' && c <= '7') {
-		    nval = (nval << 3) + (c - '0');
-                    if (--ndig)
-                        break;  /* Might have more digits */
-                } else {
-		    p--;	/* Process this character again */
-                }
-                EMIT(nval);
-                state = st_start;
-                break;
-
-	    case st_hex:
+	    case st_byte:
             case st_ucs:
-		if (nasm_isxdigit(c)) {
-		    nval = (nval << 4) + numvalue(c);
+                if ((v = numvalue_chk(c)) < base) {
+                    nval = (nval * base) + v;
                     if (--ndig)
-                        break;  /* Might have more digits */
+                        break; /* Continue processing number, no output */
+                } else if (ndig < 0) {
+                    /* End of braced sequence */
+                    if (unlikely(c != '}'))
+                        goto rewind;
                 } else {
-		    p--;	/* Process this character again */
+                    p--;        /* Reprocess terminating character */
+                    if (unlikely(p == escp+1))
+                        goto rewind; /* No digits at all received */
                 }
 
-                if (unlikely(p <= escp))
-                    EMIT(escp[-1]);
-                else if (state == st_ucs)
+                /* Emit the output */
+                if (state == st_ucs)
                     EMIT_UTF8(nval);
                 else
                     EMIT(nval);
 
                 state = st_start;
 		break;
 
+                /* Rewind an entire sequence as invalid */
+            rewind:
+                p = escp;       /* Start over at character following \ */
+                state = st_start;
+                break;
+
             default:
                 panic();
             }
@@ -421,10 +436,11 @@ size_t nasm_unquote_anystr(char *str, char **ep, const uint32_t badctl,
          * * any kind, including collapsing double quote marks.)
          * We obviously can't get here if qstart == '\"'.
          */
-        while ((c = *p++) && (c != bq))
+        p++;                    /* Skip initial quote */
+        while ((c = *p++) && c != bq)
             EMIT(c);
     } else {
-	/* Not a quoted string, just return the input... */
+	/* Not a quoted string, just return the input */
         while ((c = *p++))
             EMIT(c);
     }
@@ -435,8 +451,11 @@ size_t nasm_unquote_anystr(char *str, char **ep, const uint32_t badctl,
     if (ctlmask & badctl)
         nasm_nonfatal("control character in string not allowed here");
 
-    if (ep)
+    if (ep) {
+        /* Point at the terminating character */
 	*ep = (char *)p - 1;
+    }
+
     return (char *)q - str;
 }
 #undef EMIT
@@ -471,11 +490,7 @@ char *nasm_skip_string(const char *str)
     char bq;
     const char *p;
     char c;
-    enum unq_state {
-	st_start,
-	st_backslash,
-        st_done
-    } state;
+    enum unq_state state;
 
     bq = str[0];
     p = str+1;
@@ -515,6 +530,9 @@ char *nasm_skip_string(const char *str)
 		 * a backquote will force a return to the st_start state,
                  * and any possible multi-character state will terminate
                  * for any non-alphanumeric character.
+                 *
+                 * The only reason this is needed at all is to detect
+                 * the \` sequence.
 		 */
 		state = c ? st_start : st_done;
 		break;

doc/changes.src

@@ -92,6 +92,11 @@ It is the production version of NASM since 2025.
    size of the immediate explicitly (e.g. \c{JMP DWORD label}) has
    always worked correctly, however.
 
+\b Add support for C2y-style \c{\\o} escape sequences, braced escape
+   sequences, and as a NASM extension, decimal escape sequences
+   (\c{\\d}). See \k{strings}.
+
+
 \S{cl-3.01} Version 3.01
 
 \b A new \c{obj2} version of the \c{obj} output format, intended for

doc/lang.src

@@ -442,7 +442,6 @@ quotes allows double quotes to appear within it and vice versa); the
 contents of those are represented verbatim.  Strings enclosed in
 backquotes support C-style \c{\\}-escapes for special characters.
 
-
 The following \i{escape sequences} are recognized by backquoted strings:
 
 \c       \'          single quote (')
@@ -463,6 +462,22 @@ The following \i{escape sequences} are recognized by backquoted strings:
 \c       \u1234      4 hexadecimal digits - Unicode character
 \c       \U12345678  8 hexadecimal digits - Unicode character
 
+NASM 3.02 added the following additional sequences:
+
+\c       \o377       Up to 3 octal digits - literal byte (from C2y)
+\c       \d255       Up to 3 decimal digits - literal byte (NASM extension)
+
+Since NASM 3.02, the numeric escape sequences starting with \c{\\x},
+\c{\\o}, \c{\\d} or \c{\\u} can have their argument enclosed in curly
+braces to delimit their length, instead of terminating after a certain
+number of digits or a non-digit another character:
+
+\c       \x{FF}      Hexadecimal, literal byte
+\c       \o{377}     Octal, literal byte
+\c       \d{255}     Decimal, literal byte
+\c       \u{1234}    Unicode character
+\c       \u{12345}   Unicode character
+
 All other escape sequences are reserved.  Note that \c{\\0}, meaning a
 \c{NUL} character (ASCII 0), is a special case of the octal escape
 sequence.
@@ -471,6 +486,8 @@ sequence.
 \i{UTF-8}.  For example, the following lines are all equivalent:
 
 \c       db `\u263a`            ; UTF-8 smiley face
+\c       db `\U0000263a`        ; UTF-8 smiley face
+\c       db `\u{263a}`          ; UTF-8 smiley face
 \c       db `\xe2\x98\xba`      ; UTF-8 smiley face
 \c       db 0E2h, 098h, 0BAh    ; UTF-8 smiley face
 

include/nasmlib.h

@@ -273,6 +273,22 @@ static inline unsigned int numvalue(unsigned char c)
     return c >= 'a' ? c - 'a' + 10 : c - '0';
 }
 
+/* The same except returns -1U for non-digits, so it can be directly
+ * compared against the base used to test for validity. */
+static inline unsigned int numvalue_chk(unsigned char c)
+{
+    unsigned int v;
+    v = c - '0';
+    if (v < 10)
+        return v;
+
+    v = (c | 0x20) - 'a';
+    if (v < 26)
+        return v + 10;
+
+    return -1U;
+}
+
 /*
  * Convert a string into a number, using NASM number rules. Sets
  * `*error' to true if an error occurs, and false otherwise.

test/bracedquote.asm

@@ -0,0 +1,21 @@
+	db "Have a day!!", 13, 10
+	db 13, 10
+	;; All these should produce U+1F610 in UTF-8
+	db `\U0001f610`
+	db `\u{1f610}`
+	db `\u{0001f610}`
+	db `\x{f0}\x{9f}\x{98}\x{90}`
+	db `\xf0\x9f\x98\x90`
+	db `\d240\d159\d152\d144`
+	db `\d{240}\d{0159}\d{152}\d{144}`
+	db `\360\237\230\220`
+	db `\o{360}\o{0237}\o{230}\o{00220}`
+	db `\o360\o237\o230\o220`
+
+	db `\U0001F610`
+	db `\u{1F610}`
+	db `\u{0001F610}`
+	db `\x{F0}\x{9F}\x{98}\x{90}`
+	db `\xF0\x9F\x98\x90`
+
+	db `\r\n`