Use ASCII case folding for :lang() and attribute values

cssselect/xpath.py

@@ -440,7 +440,7 @@ def xpath_attrib(self, selector: Attrib) -> XPathExpr:
         if selector.value is None:
             value = None
         elif self.lower_case_attribute_values:
-            value = cast("str", selector.value.value).lower()
+            value = ascii_lower(cast("str", selector.value.value))
         else:
             value = selector.value.value
         if selector.flag == "i" and value:
@@ -927,7 +927,9 @@ def xpath_lang_function(self, xpath: XPathExpr, function: Function) -> XPathExpr
             )
         value = function.arguments[0].value
         assert value
-        arg = self.xpath_literal(value.lower() + "-")
+        # Language-range matching is ASCII case-insensitive (RFC 4647),
+        # like the translate() on the document side of the comparison.
+        arg = self.xpath_literal(ascii_lower(value) + "-")
         return xpath.add_condition(
             "ancestor-or-self::*[@lang][1][starts-with(concat("
             # XPath 1.0 has no lower-case function...

tests/test_cssselect.py

@@ -1045,6 +1045,9 @@ class LowerValues(GenericTranslator):
             lower_case_attribute_values = True
 
         assert LowerValues().css_to_xpath("[Foo=BAR]", prefix="") == "*[@Foo = 'bar']"
+        # The folding is ASCII-only: str.lower() would turn U+212A
+        # (KELVIN SIGN) into ASCII "k".
+        assert LowerValues().css_to_xpath('[foo="K"]', prefix="") == "*[@foo = 'K']"
 
         # A member of an :is()/:where() selector list that translates to a
         # path (rather than a predicate) cannot be embedded in the outer
@@ -1386,6 +1389,17 @@ def test_html_non_ascii_attribute_names(self) -> None:
             == "descendant-or-self::*[attribute::*[name() = 'data-ä'] = 'yes']"
         )
 
+    def test_html_non_ascii_lang(self) -> None:
+        # Language-range matching is ASCII case-insensitive (RFC 4647);
+        # str.lower() folds U+212A (KELVIN SIGN) into ASCII "k".
+        xpath = HTMLTranslator().css_to_xpath
+        assert xpath(":lang(K)") == (
+            "descendant-or-self::*[ancestor-or-self::*[@lang][1]"
+            "[starts-with(concat(translate(@lang, "
+            "'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), "
+            "'-'), 'K-')]]"
+        )
+
     def test_select(self) -> None:
         document = etree.fromstring(HTML_IDS)
         sort_key = {el: count for count, el in enumerate(document.iter())}.__getitem__