diff --git a/cssselect/xpath.py b/cssselect/xpath.py index 8ef5d7c..88de261 100644 --- a/cssselect/xpath.py +++ b/cssselect/xpath.py @@ -314,6 +314,7 @@ def xpath_combinedselector(self, combined: CombinedSelector) -> XPathExpr: return method(self.xpath(combined.selector), self.xpath(combined.subselector)) def xpath_negation(self, negation: Negation) -> XPathExpr: + """Translate ``:not()``.""" xpath = self.xpath(negation.selector) condition = self._xpath_match_condition(negation.subselector) if condition is None: @@ -341,6 +342,7 @@ def _xpath_match_condition(self, selector: Tree) -> str | None: return sub_xpath.condition or None def xpath_relation(self, relation: Relation) -> XPathExpr: + """Translate ``:has()``.""" xpath = self.xpath(relation.selector) combinator = relation.combinator subselector = relation.subselector @@ -355,11 +357,13 @@ def xpath_relation(self, relation: Relation) -> XPathExpr: return method(xpath, right) def xpath_matching(self, matching: Matching) -> XPathExpr: + """Translate ``:is()`` and its alias ``:matches()``.""" return self._xpath_add_selector_list_condition( self.xpath(matching.selector), matching.selector_list ) def xpath_specificityadjustment(self, matching: SpecificityAdjustment) -> XPathExpr: + """Translate ``:where()``.""" return self._xpath_add_selector_list_condition( self.xpath(matching.selector), matching.selector_list ) diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000..9e7cad0 --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,7 @@ +/* sphinx_rtd_theme forces nowrap on table cells and relies on horizontal + scrolling instead; wrap long cell content, e.g. in the selector support + table, so it stays fully visible without scrolling. */ +.wy-table-responsive table td, +.wy-table-responsive table th { + white-space: normal; +} diff --git a/docs/conf.py b/docs/conf.py index da3f023..353bcae 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -122,7 +122,8 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] +html_static_path = ["_static"] +html_css_files = ["custom.css"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/docs/index.rst b/docs/index.rst index c86583c..4007ca9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -73,54 +73,86 @@ Supported selectors =================== This library implements CSS3 selectors as described in `the W3C specification -`_. -In this context however, there is no interactivity or history of visited links. -Therefore, these pseudo-classes are accepted but never match anything: - -* ``:hover`` -* ``:active`` -* ``:focus`` -* ``:target`` -* ``:visited`` - -Additionally, these depend on document knowledge and only have a useful -implementation in :class:`HTMLTranslator`. In :class:`GenericTranslator`, -they never match: - -* ``:link`` -* ``:enabled`` -* ``:disabled`` -* ``:checked`` - -These applicable pseudo-classes are not yet implemented: - -* ``*:first-of-type``, ``*:last-of-type``, ``*:nth-of-type``, - ``*:nth-last-of-type``, ``*:only-of-type``. All of these work when - you specify an element type, but not with ``*`` - -On the other hand, *cssselect* supports some selectors that are not -in the Level 3 specification. - -These parts of the Level 4 specification are supported (note that a large part -of the Level 4 additions is not applicable to cssselect similarly to ``:hover`` -or not representable in XPath 1.0 so the complete specification is unlikely to -be implemented): - -* The ``:scope`` pseudo-class. Limitation: it can only be used at a start of a - selector. -* The ``:is()`` and ``:where()`` pseudo-classes. Limitation: their arguments - are a comma-separated list of *compound selectors*; combinators are not - allowed (e.g. ``:is(a b)`` or ``:is(a > b)``). ``:not()`` and ``:scope`` are - also rejected inside them, while ``:has()`` is supported (e.g. - ``:is(:has(> a))``). -* The ``:has()`` pseudo-class. Limitation: it takes a single argument, made of - an optional leading combinator (``>``, ``+`` or ``~``) followed by one - *compound selector* built only from type, class and universal selectors - (e.g. ``:has(> a.important)``). Anything else is unsupported, e.g. an ID - (``:has(#id)``), or a selector list (``:has(a, b)``). -* The ``:not()`` pseudo-class with a *complex selector* argument, e.g. - ``:not(a.important[rel] > b)``. Limitation: it takes a single argument, so a - selector list is unsupported (e.g. ``:not(a, b)``). +`_, plus a few parts +of Level 4. The table below lists every selector *cssselect* recognizes and +whether it can be translated to XPath 1.0; selectors it does not recognize at +all raise :class:`SelectorSyntaxError`, and recognized-but-untranslatable ones +raise :class:`ExpressionError`. + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - Selector + - Support + * - ``*``, ``e``, ``ns|e``, ``*|e``, ``|e`` + - Yes + * - ``e#id``, ``e.class`` + - Yes + * - ``[attr]``, ``[attr=val]``, ``[attr~=val]``, ``[attr|=val]``, + ``[attr^=val]``, ``[attr$=val]``, ``[attr*=val]``, + ``[namespace|attr...]`` + - Yes + * - ``e f``, ``e > f``, ``e + f``, ``e ~ f`` + - Yes + * - ``e, f`` (grouping) + - Yes + * - ``:first-child``, ``:last-child``, ``:only-child`` + - Yes + * - ``e:first-of-type``, ``e:last-of-type``, ``e:only-of-type``, + ``e:nth-of-type()``, ``e:nth-last-of-type()`` + - Yes, when an element type is given + * - ``*:first-of-type``, ``*:last-of-type``, ``*:only-of-type``, + ``*:nth-of-type()``, ``*:nth-last-of-type()`` + - No: counting same-type siblings needs a concrete type, and ``*`` + does not have one + * - ``:nth-child()``, ``:nth-last-child()`` + - Yes, including the ``even``, ``odd`` and ``an+b`` forms + * - ``:empty``, ``:root`` + - Yes + * - ``:lang()`` + - Yes + * - ``:not(compound or complex selector)`` + - Yes, for a single argument; ``:not(a, b)`` (a selector list) is not + supported + * - ``:hover``, ``:active``, ``:focus``, ``:target``, ``:visited`` + - Parsed, but never matches: there is no interactivity or link + history to base this on + * - ``:link``, ``:enabled``, ``:disabled``, ``:checked`` + - Never matches in :class:`GenericTranslator`; matched based on + HTML-specific elements and attributes in :class:`HTMLTranslator` + * - ``::before``, ``::after``, ``::first-line``, ``::first-letter``, + ``::marker``, functional pseudo-elements + - Parsed into :attr:`Selector.pseudo_element`, but rejected by + :meth:`~GenericTranslator.css_to_xpath`: XPath has no notion of a + pseudo-element, so it is up to the caller to handle these + * - ``:scope`` + - Yes, only at the start of a selector (Level 4) + * - ``:is()``, ``:where()``, and their alias ``:matches()`` + - Yes, for a comma-separated list of *compound* selectors; a + combinator (``:is(a b)``), ``:not()`` or ``:scope`` inside the + arguments is not supported, but ``:has()`` is (Level 4) + * - ``:has()`` + - Yes, for a single argument made of an optional leading combinator + (``>``, ``+`` or ``~``) followed by one *compound selector* built + only from type, class and universal selectors, e.g. + ``:has(> a.important)``; anything else, such as an ID + (``:has(#id)``) or a selector list (``:has(a, b)``), is not + supported (Level 4) + * - Case-insensitive (``i``) and case-sensitive (``s``) attribute value + flags, ``:nth-child(An+B of S)``, ``:dir()``, ``:focus-within``, + ``:focus-visible``, ``:default``, ``:placeholder-shown``, + ``:read-only``/``:read-write``, ``:required``/``:optional``, + ``:valid``/``:invalid``/``:in-range``/``:out-of-range``, + ``:current``/``:past``/``:future``, + ``:nth-col()``/``:nth-last-col()``, ``:host()``/``:host-context()``, + ``::selection``, ``::placeholder``, ``::part()``, ``::slotted()`` + - Not implemented; several of these need document or runtime state + cssselect has no access to, or a shadow DOM concept that has no + XPath equivalent + +This is not an exhaustive list of every Level 4 (or later) selector; a +selector that is not mentioned here at all is not implemented either. These are non-standard extensions: @@ -151,6 +183,201 @@ These are non-standard extensions: It should not be a problem anymore. +Examples +======== + +Every example below runs as a doctest alongside the test suite, so it stays +correct as the translation logic changes. + +.. sourcecode:: pycon + + >>> from cssselect import GenericTranslator + >>> def xpath(css, translator=GenericTranslator()): + ... return translator.css_to_xpath(css, prefix="") + +Type, universal, namespace, ID and class selectors: + +.. sourcecode:: pycon + + >>> xpath("*") + '*' + >>> xpath("e") + 'e' + >>> xpath("ns|e") + 'ns:e' + >>> xpath("e#myid") + "e[@id = 'myid']" + >>> xpath("e.warning") + "e[@class and contains(concat(' ', normalize-space(@class), ' '), ' warning ')]" + +Attribute selectors: + +.. sourcecode:: pycon + + >>> xpath("e[foo]") + 'e[@foo]' + >>> xpath('e[foo="bar"]') + "e[@foo = 'bar']" + >>> xpath('e[foo~="bar"]') + "e[@foo and contains(concat(' ', normalize-space(@foo), ' '), ' bar ')]" + >>> xpath('e[foo^="bar"]') + "e[@foo and starts-with(@foo, 'bar')]" + >>> xpath('e[foo$="bar"]') + "e[@foo and substring(@foo, string-length(@foo)-2) = 'bar']" + >>> xpath('e[foo*="bar"]') + "e[@foo and contains(@foo, 'bar')]" + >>> xpath('e[foo|="en"]') + "e[@foo and (@foo = 'en' or starts-with(@foo, 'en-'))]" + >>> xpath("e[foo!=bar]") # non-standard extension + "e[not(@foo) or @foo != 'bar']" + +Combinators and grouping: + +.. sourcecode:: pycon + + >>> xpath("e f") + 'e/descendant-or-self::*/f' + >>> xpath("e > f") + 'e/f' + >>> xpath("e + f") + 'e/following-sibling::*[(self::f) and (position() = 1)]' + >>> xpath("e ~ f") + 'e/following-sibling::f' + >>> xpath("e, f") + 'e | f' + +Structural pseudo-classes: + +.. sourcecode:: pycon + + >>> xpath("e:first-child") + 'e[count(preceding-sibling::*) = 0]' + >>> xpath("e:nth-child(3n+2)") + 'e[(count(preceding-sibling::*) >= 1) and ((count(preceding-sibling::*) +2) mod 3 = 0)]' + >>> xpath("e:only-child") + 'e[count(preceding-sibling::*) = 0 and count(following-sibling::*) = 0]' + >>> xpath("e:empty") + 'e[not(*) and not(string-length())]' + >>> xpath("e:root") + 'e[not(parent::*)]' + +``:not()``, ``:is()``, ``:where()`` and ``:has()``: + +.. sourcecode:: pycon + + >>> xpath("e:not(a > b)") + 'e[not(self::b and parent::*[self::a])]' + >>> xpath("e:where(foo, bar)") + 'e[(self::foo) or (self::bar)]' + >>> xpath("e:has(> f)") + 'e[./f]' + >>> xpath("div:has(bar.foo)") + "div[descendant::bar[@class and contains(concat(' ', normalize-space(@class), ' '), ' foo ')]]" + +``:lang()`` and the non-standard ``:contains()``: + +.. sourcecode:: pycon + + >>> xpath("e:lang(en)") + "e[lang('en')]" + >>> xpath('e:contains("foo")') + "e[contains(., 'foo')]" + + +How selectors become XPath +=========================== + +Translating a selector is two independent steps, one per module. + +Parsing +------- + +In :file:`cssselect/parser.py`, :func:`parse` tokenizes the CSS source +(``tokenize()``) and feeds the resulting ``Token`` stream to a small +recursive-descent parser: ``parse_selector_group()`` splits a +comma-separated group, ``parse_selector()`` handles combinators, and +``parse_simple_selector()`` handles everything that can appear in a single +compound selector (type, ``#id``, ``.class``, ``[attr]``, and +pseudo-classes/elements). Each construct becomes one node of a *parsed +tree*, wrapped in a :class:`Selector`: + +.. list-table:: + :header-rows: 1 + + * - CSS + - Parsed as + * - ``e``, ``*``, ``ns|e`` + - ``Element`` + * - ``#id`` + - ``Hash`` + * - ``.class`` + - ``Class`` + * - ``[attr...]`` + - ``Attrib`` + * - ``:name`` (no arguments) + - ``Pseudo`` + * - ``:name(...)`` (other than the ones below) + - ``Function`` + * - ``:not(...)`` + - ``Negation`` + * - ``:has(...)`` + - ``Relation`` + * - ``:is(...)``, ``:matches(...)`` + - ``Matching`` + * - ``:where(...)`` + - ``SpecificityAdjustment`` + * - ``e f``, ``e > f``, ``e + f``, ``e ~ f`` + - ``CombinedSelector`` + +Translation +----------- + +In :file:`cssselect/xpath.py`, ``GenericTranslator.xpath()`` walks that tree +and turns it into an ``XPathExpr``, a small builder that accumulates an +XPath ``path`` (e.g. ``foo/descendant-or-self::*/``), an ``element`` node +test, and a predicate ``condition``, joined together into a string. It +dispatches on the parsed node's class name, lower-cased and prefixed with +``xpath_``, e.g. an ``Attrib`` node is handled by ``xpath_attrib()``. + +A handful of node types dispatch a second time, by name, to keep one method +per CSS construct instead of one large method with a chain of ``if``\\ s. +The method name is always ``xpath_`` + that name (``-`` becomes ``_``) + +a suffix identifying the kind of dispatch: + +* ``Pseudo`` (``:name``) and ``Function`` (``:name(...)``) dispatch on the + pseudo-class or function name, e.g. ``:first-child`` and ``:nth-child()`` + are handled by ``xpath_first_child_pseudo()`` and + ``xpath_nth_child_function()``. These two are open-ended: adding a + pseudo-class or function is adding a method with the matching name. +* ``CombinedSelector`` and ``Relation`` (whose subselector can carry its own + leading combinator, e.g. ``:has(> a)``) dispatch on the combinator, mapped + by name in ``GenericTranslator.combinator_mapping``, e.g. ``>`` becomes + ``xpath_child_combinator()`` and ``xpath_relation_child_combinator()`` + respectively. +* ``Attrib`` dispatches on the attribute operator, mapped by name in + ``GenericTranslator.attribute_operator_mapping``, e.g. ``^=`` becomes + ``xpath_attrib_prefixmatch()``. + +``xpath_negation()`` (``:not()``), ``xpath_matching()`` +(``:is()``/``:matches()``) and ``xpath_specificityadjustment()`` +(``:where()``) are the exceptions: their CSS name does not appear in the +Python method name, since none of them dispatch further by name. + +``:not()`` is also the one construct not translated by walking left to +right: matching e.g. ``a:not(b > c)`` means testing, on the context node +itself, whether it is a ``c`` whose parent is a ``b`` — the reverse of how +``b > c`` alone would be read. ``_xpath_match_condition()`` does this by +walking the ``:not()`` argument right to left, turning each combinator into +the matching reverse axis from +``GenericTranslator._reverse_combinator_mapping`` (e.g. ``>`` becomes +``parent::*``). + +:class:`HTMLTranslator` overrides a handful of these hooks (``:link``, +``:checked``, ``:enabled``, ``:disabled`` and ``:lang()``) with +HTML-specific logic, and three case-folding flags, but otherwise reuses +everything above. + + Customizing the translation ===========================