[Go] Add Unicode Bypass Validation query, test and help file

go/ql/lib/semmle/go/security/UnicodeBypassValidationCustomizations.qll

@@ -0,0 +1,29 @@
+/**
+ * Provides default sources, sinks and sanitizers for detecting
+ * "Unicode transformation"
+ * vulnerabilities, as well as extension points for adding your own.
+ */
+
+private import go
+
+/**
+ * Provides default sources, sinks and sanitizers for detecting
+ * "Unicode transformation"
+ * vulnerabilities, as well as extension points for adding your own.
+ */
+module UnicodeBypassValidation {
+  /**
+   * A data flow source for "Unicode transformation" vulnerabilities.
+   */
+  abstract class Source extends DataFlow::Node { }
+
+  /**
+   * A data flow sink for "Unicode transformation" vulnerabilities.
+   */
+  abstract class Sink extends DataFlow::Node { }
+
+  /**
+   * A sanitizer for "Unicode transformation" vulnerabilities.
+   */
+  abstract class Sanitizer extends DataFlow::Node { }
+}

go/ql/lib/semmle/go/security/UnicodeBypassValidationQuery.qll

@@ -0,0 +1,179 @@
+/**
+ * Provides a taint-tracking configuration for detecting "Unicode transformation mishandling" vulnerabilities.
+ */
+
+private import go
+import semmle.go.Concepts
+import semmle.go.dataflow.DataFlow
+import semmle.go.dataflow.TaintTracking
+import UnicodeBypassValidationCustomizations::UnicodeBypassValidation
+
+/** A state signifying that a logical validation has not been performed. */
+class PreValidation extends DataFlow::FlowState {
+  PreValidation() { this = "PreValidation" }
+}
+
+/** A state signifying that a logical validation has been performed. */
+class PostValidation extends DataFlow::FlowState {
+  PostValidation() { this = "PostValidation" }
+}
+
+private predicate indexCheck(DataFlow::Node g, Expr e, boolean outcome) {
+  exists(DataFlow::CallNode cn, DataFlow::EqualityTestNode etn |
+    g = etn and
+    DataFlow::localFlow(cn.getResult(), etn.getAnOperand()) and
+    cn.getTarget()
+        .hasQualifiedName("strings",
+          [
+            "Index", "IndexAny", "IndexByte", "IndexFunc", "IndexRune", "LastIndex", "LastIndexAny",
+            "LastIndexByte", "LastIndexFunc",
+          ]) and
+    cn.getArgument(0).asExpr() = e and
+    etn.getAnOperand().getIntValue() = -1 and
+    outcome = etn.getPolarity()
+  )
+}
+
+private predicate countCheck(DataFlow::Node g, Expr e, boolean outcome) {
+  exists(
+    DataFlow::RelationalComparisonNode cmp, DataFlow::CallNode cn, DataFlow::Node zero,
+    DataFlow::Node r
+  |
+    g = cmp and
+    DataFlow::localFlow(cn.getResult(), r) and
+    cn.getTarget().hasQualifiedName("strings", "Count") and
+    cn.getArgument(1).asExpr() = e and
+    zero.getNumericValue() = 0 and
+    cmp.leq(outcome, r, zero, 0)
+  )
+}
+
+private predicate boolCheck(DataFlow::Node cn, Expr e, boolean outcome) {
+  cn.(DataFlow::CallNode)
+      .getTarget()
+      .hasQualifiedName("strings",
+        ["Contains", "ContainsAny", "ContainsRune", "HasPrefix", "HasSuffix", "EqualFold"]) and
+  cn.(DataFlow::CallNode).getArgument(0).asExpr() = e and
+  outcome = false
+}
+
+private predicate compareCheck(DataFlow::Node cn, Expr e, boolean outcome) {
+  cn.(DataFlow::CallNode)
+      .getTarget()
+      .hasQualifiedName("strings", ["Compare", "CompareFold", "ComparePrefix", "CompareSuffix"]) and
+  cn.(DataFlow::CallNode).getArgument(0).asExpr() = e and
+  outcome = false
+}
+
+private predicate regexMatchCheck(DataFlow::Node cn, Expr e, boolean outcome) {
+  cn.(DataFlow::CallNode).getTarget() instanceof RegexpMatchFunction and
+  cn.(DataFlow::CallNode).getArgument(0).asExpr() = e and
+  outcome = false
+}
+
+/**
+ * A use of a variable guarded by a call to `Index`, `ContainsAny`, Regex match functions
+ * or similar, in a context suggesting it has been validated to not contain a particular character.
+ */
+class UntrustedUnicodeCharChecks extends DataFlow::Node {
+  UntrustedUnicodeCharChecks() {
+    this = DataFlow::BarrierGuard<indexCheck/3>::getABarrierNode()
+    or
+    this = DataFlow::BarrierGuard<countCheck/3>::getABarrierNode()
+    or
+    this = DataFlow::BarrierGuard<boolCheck/3>::getABarrierNode()
+    or
+    this = DataFlow::BarrierGuard<regexMatchCheck/3>::getABarrierNode()
+    or
+    this = DataFlow::BarrierGuard<compareCheck/3>::getABarrierNode()
+  }
+}
+
+/**
+ * A taint-tracking configuration for detecting "Unicode transformation mishandling" vulnerabilities.
+ *
+ * This configuration uses two flow states, `PreValidation` and `PostValidation`,
+ * to track the requirement that a logical validation has been performed before the Unicode Transformation.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "UnicodeBypassValidation" }
+
+  override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
+    (source instanceof UntrustedFlowSource or source instanceof Source) and
+    state instanceof PreValidation
+  }
+
+  override predicate isSanitizer(DataFlow::Node sanitizer, DataFlow::FlowState state) {
+    sanitizer instanceof Sanitizer and
+    state instanceof PostValidation
+  }
+
+  override predicate isAdditionalTaintStep(
+    DataFlow::Node nodeFrom, DataFlow::FlowState stateFrom, DataFlow::Node nodeTo,
+    DataFlow::FlowState stateTo
+  ) {
+    (
+      exists(DataFlow::CallNode cn |
+        cn.getACalleeIncludingExternals().asFunction() instanceof EscapeFunction and
+        nodeFrom = cn.getAnArgument() and
+        nodeTo = cn.getResult()
+      )
+      or
+      exists(DataFlow::CallNode cn |
+        cn.getACalleeIncludingExternals().asFunction() instanceof RegexpReplaceFunction and
+        nodeFrom = cn.getAnArgument() and
+        nodeTo = cn.getResult()
+      )
+      or
+      exists(DataFlow::CallNode cn |
+        cn.getTarget().hasQualifiedName("strings", "Cut") and
+        nodeFrom = cn.getArgument(0) and
+        nodeTo = cn.getResult([0, 1])
+      )
+      or
+      exists(DataFlow::CallNode cn |
+        cn.getTarget().hasQualifiedName("strings", ["CutPrefix", "CutSuffix"]) and
+        nodeFrom = cn.getArgument(0) and
+        nodeTo = cn.getResult(0)
+      )
+      or
+      exists(DataFlow::CallNode cn |
+        cn.getTarget()
+            .hasQualifiedName("strings",
+              [
+                "Fields", "FieldsFunc", "Replace", "ReplaceAll", "Split", "SplitAfter",
+                "SplitAfterN", "SplitN", "ToLower", "ToLowerSpecial", "ToTitle", "ToTitleSpecial",
+                "ToUpper", "ToUpperSpecial", "Trim", "TrimFunc", "TrimLeft", "TrimLeftFunc",
+                "TrimPrefix", "TrimRight", "TrimRightFunc", "TrimSpace", "TrimSuffix",
+              ]) and
+        nodeFrom = cn.getArgument(0) and
+        nodeTo = cn.getAResult()
+      )
+    ) and
+    stateFrom instanceof PreValidation and
+    stateTo instanceof PostValidation
+  }
+
+  /*
+   * A Unicode Tranformation is considered a sink when the form algorithm used is for Unicode normalization (NFC, NFKC, etc) and one of the two scenarios happens:
+   *  - The flow went through a call to an Escape function, Regex Replace function, or a String manipulation function.
+   *  - The Unicode normalisation was guarded by a check that the input does not contain a particular character either using Regex match functions or String checks functions like Index/Contains functions.
+   */
+
+  override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
+    (
+      exists(string unicodeNorm, DataFlow::MethodCallNode cn |
+        unicodeNorm = package("golang.org/x/text", "unicode/norm") and
+        cn.getTarget().hasQualifiedName(unicodeNorm, "Form", "String") and
+        sink = cn.getArgument(0)
+      )
+      or
+      sink instanceof Sink
+    ) and
+    (
+      state instanceof PostValidation
+      or
+      sink instanceof UntrustedUnicodeCharChecks and state instanceof PreValidation
+    )
+  }
+}

go/ql/src/experimental/CWE-176/UnicodeBypassValidation.qhelp

@@ -0,0 +1,34 @@
+<!DOCTYPE qhelp PUBLIC "-//Semmle//qhelp//EN" "qhelp.dtd">
+<qhelp>
+    <overview>
+        <p>
+            If ever a unicode tranformation is performed after some security checks or logical
+            validation, those
+            checks could be bypassed due to a potential Unicode characters collision.
+            The validation of concern are any character escaping, any regex validation or any string
+            verification.
+        </p>
+    </overview>
+    <recommendation>
+        <p> Perform a Unicode normalization before the logical validation. </p>
+    </recommendation>
+    <example>
+
+        <p> The following example showcases the bypass of all checks performed by <code>
+            html.EscapeString()</code> due to a post-unicode normalization.</p>
+        <p>For instance: the character U+FE64 (<code>﹤</code>) is not filtered-out by the flask
+            escape function. But due to the Unicode normalization, the character is transformed and
+            would become U+003C (<code> &lt; </code> ).</p>
+
+        <sample src="example.go" />
+
+    </example>
+    <references>
+        <li> Research study: <a
+                href="https://gosecure.github.io/presentations/2021-02-unicode-owasp-toronto/philippe_arteau_owasp_unicode_v4.pdf">
+            Unicode vulnerabilities that could bYte you
+            </a> and <a
+                href="https://gosecure.github.io/unicode-pentester-cheatsheet/">Unicode pentest
+            cheatsheet</a>. </li>
+    </references>
+</qhelp>

go/ql/src/experimental/CWE-176/UnicodeBypassValidation.ql

@@ -0,0 +1,24 @@
+/**
+ * @name Bypass Logical Validation Using Unicode Characters
+ * @description A Unicode transformation is using a remote user-controlled data. The transformation is a Unicode normalization . In all cases, the security measures implemented or the logical validation performed to escape any injection characters, to validate using regex patterns or to perform string-based checks, before the Unicode transformation are **bypassable** by special Unicode characters.
+ * @kind path-problem
+ * @id go/unicode-bypass-validation
+ * @precision high
+ * @problem.severity error
+ * @tags security
+ *       experimental
+ *       external/cwe/cwe-176
+ *       external/cwe/cwe-179
+ *       external/cwe/cwe-180 
+ */
+
+import go
+import semmle.go.security.UnicodeBypassValidationQuery
+import DataFlow::PathGraph
+
+from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
+where config.hasFlowPath(source, sink)
+select sink.getNode(), source, sink,
+  "This $@ processes unsafely $@ and any logical validation in-between could be bypassed using special Unicode characters.",
+  sink.getNode(), "Unicode transformation (Unicode normalization)", source.getNode(),
+  "remote user-controlled data"

go/ql/src/experimental/CWE-176/example.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"fmt"
+	"html"
+	"net/http"
+
+	"golang.org/x/text/unicode/norm"
+)
+
+func main() {}
+
+func bad() {
+	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
+
+		unicode_input := req.URL.Query().Get("unicode_input")
+		escaped := html.EscapeString(unicode_input)
+		unicode_norm := norm.NFKC.String(escaped)
+		fmt.Println(w, "Results: %q", unicode_norm)
+	})
+}

go/ql/test/experimental/CWE-176/UnicodeBypassValidation.expected

@@ -0,0 +1,40 @@
+edges
+| example_bad.go:17:20:17:26 | selection of URL | example_bad.go:17:20:17:34 | call to Query |
+| example_bad.go:17:20:17:34 | call to Query | example_bad.go:17:20:17:55 | call to Get |
+| example_bad.go:17:20:17:55 | call to Get | example_bad.go:18:32:18:44 | unicode_input |
+| example_bad.go:18:14:18:45 | call to EscapeString | example_bad.go:19:36:19:42 | escaped |
+| example_bad.go:18:32:18:44 | unicode_input | example_bad.go:18:14:18:45 | call to EscapeString |
+| example_bad.go:25:20:25:26 | selection of URL | example_bad.go:25:20:25:34 | call to Query |
+| example_bad.go:25:20:25:34 | call to Query | example_bad.go:25:20:25:55 | call to Get |
+| example_bad.go:25:20:25:55 | call to Get | example_bad.go:27:37:27:49 | unicode_input |
+| example_bad.go:35:20:35:26 | selection of URL | example_bad.go:35:20:35:34 | call to Query |
+| example_bad.go:35:20:35:34 | call to Query | example_bad.go:35:20:35:55 | call to Get |
+| example_bad.go:35:20:35:55 | call to Get | example_bad.go:37:37:37:49 | unicode_input |
+| example_bad.go:45:20:45:26 | selection of URL | example_bad.go:45:20:45:34 | call to Query |
+| example_bad.go:45:20:45:34 | call to Query | example_bad.go:45:20:45:55 | call to Get |
+| example_bad.go:45:20:45:55 | call to Get | example_bad.go:47:37:47:49 | unicode_input |
+nodes
+| example_bad.go:17:20:17:26 | selection of URL | semmle.label | selection of URL |
+| example_bad.go:17:20:17:34 | call to Query | semmle.label | call to Query |
+| example_bad.go:17:20:17:55 | call to Get | semmle.label | call to Get |
+| example_bad.go:18:14:18:45 | call to EscapeString | semmle.label | call to EscapeString |
+| example_bad.go:18:32:18:44 | unicode_input | semmle.label | unicode_input |
+| example_bad.go:19:36:19:42 | escaped | semmle.label | escaped |
+| example_bad.go:25:20:25:26 | selection of URL | semmle.label | selection of URL |
+| example_bad.go:25:20:25:34 | call to Query | semmle.label | call to Query |
+| example_bad.go:25:20:25:55 | call to Get | semmle.label | call to Get |
+| example_bad.go:27:37:27:49 | unicode_input | semmle.label | unicode_input |
+| example_bad.go:35:20:35:26 | selection of URL | semmle.label | selection of URL |
+| example_bad.go:35:20:35:34 | call to Query | semmle.label | call to Query |
+| example_bad.go:35:20:35:55 | call to Get | semmle.label | call to Get |
+| example_bad.go:37:37:37:49 | unicode_input | semmle.label | unicode_input |
+| example_bad.go:45:20:45:26 | selection of URL | semmle.label | selection of URL |
+| example_bad.go:45:20:45:34 | call to Query | semmle.label | call to Query |
+| example_bad.go:45:20:45:55 | call to Get | semmle.label | call to Get |
+| example_bad.go:47:37:47:49 | unicode_input | semmle.label | unicode_input |
+subpaths
+#select
+| example_bad.go:19:36:19:42 | escaped | example_bad.go:17:20:17:26 | selection of URL | example_bad.go:19:36:19:42 | escaped | This $@ processes unsafely $@ and any logical validation in-between could be bypassed using special Unicode characters. | example_bad.go:19:36:19:42 | escaped | Unicode transformation (Unicode normalization) | example_bad.go:17:20:17:26 | selection of URL | remote user-controlled data |
+| example_bad.go:27:37:27:49 | unicode_input | example_bad.go:25:20:25:26 | selection of URL | example_bad.go:27:37:27:49 | unicode_input | This $@ processes unsafely $@ and any logical validation in-between could be bypassed using special Unicode characters. | example_bad.go:27:37:27:49 | unicode_input | Unicode transformation (Unicode normalization) | example_bad.go:25:20:25:26 | selection of URL | remote user-controlled data |
+| example_bad.go:37:37:37:49 | unicode_input | example_bad.go:35:20:35:26 | selection of URL | example_bad.go:37:37:37:49 | unicode_input | This $@ processes unsafely $@ and any logical validation in-between could be bypassed using special Unicode characters. | example_bad.go:37:37:37:49 | unicode_input | Unicode transformation (Unicode normalization) | example_bad.go:35:20:35:26 | selection of URL | remote user-controlled data |
+| example_bad.go:47:37:47:49 | unicode_input | example_bad.go:45:20:45:26 | selection of URL | example_bad.go:47:37:47:49 | unicode_input | This $@ processes unsafely $@ and any logical validation in-between could be bypassed using special Unicode characters. | example_bad.go:47:37:47:49 | unicode_input | Unicode transformation (Unicode normalization) | example_bad.go:45:20:45:26 | selection of URL | remote user-controlled data |