From da51be99326c8d7f84d619e99952111e4507ae24 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 1 Jul 2026 18:37:43 -0400 Subject: [PATCH 1/2] TIKA-4766: replace FetchAndParseReply fields map with a typed Document contract One small, stable proto instead of a message per source format. Contract (tika-grpc-api, ~200 lines of proto): - Document: content as a structured markdown block tree (headings, paragraphs, lists, tables, code blocks, inline runs -- CommonMark + GFM, the same markdown ToMarkdownContentHandler already renders), plus `markdown` as the authoritative rendered form. - DocumentMetadata: a small bounded set of typed common fields grouped by concern, not by source format (title/authors/keywords/languages, created and modified as Timestamps, page/word/character counts, dimensions, rights). - Tagged tail: `extra` carries every remaining Tika key losslessly and multivalue-preserving, typed only where Tika's own Property declares a type (integer/real/boolean/date) and string otherwise -- never guessed. - `embedded` recurses: a parent PDF and an embedded image are each a fully typed Document. - `format_category` is a cheap routing hint; cross-cutting concerns such as Creative Commons rights coexist with it rather than fighting a oneof. Format specifics live in per-parser DocumentTransformer code (tika-grpc-mapper), not in the wire contract: adding or changing a parser never touches the proto, and metadata churn lands in the mapper and the tagged tail, so a new or renamed metadata key never forces a client rebuild. Server (tika-grpc): - FetchAndParseReply.document (field 5) replaces the fields map (field 2, now reserved). TikaGrpcServerImpl maps parse output via DocumentBuilder. Modules: tika-grpc-api (proto + generated messages + bundled FileDescriptorSet), tika-grpc-mapper (DocumentBuilder, per-format transformers, markdown block-tree builder), both listed in tika-bom. Tests: transformer tests against real parse fixtures per format, block tree round-trip tests, DocumentBuilder envelope/status/embedded tests, server tests updated to read FetchAndParseReply.document. --- pom.xml | 2 + tika-bom/pom.xml | 15 + .../pipes/filesystem/HandlerTypeTest.java | 4 +- tika-grpc-api/README.md | 55 +++ tika-grpc-api/buf.yaml | 29 ++ tika-grpc-api/pom.xml | 121 ++++++ .../org/apache/tika/grpc/v1/document.proto | 208 ++++++++++ tika-grpc-mapper/docs/EXTENSIONS.md | 43 ++ .../docs/SOURCE_DESTINATION_MAPPING.md | 369 ++++++++++++++++++ .../docs/TIKA_INTERFACE_MAPPING.md | 253 ++++++++++++ tika-grpc-mapper/pom.xml | 213 ++++++++++ .../tika/grpc/mapper/DocumentBuilder.java | 149 +++++++ .../content/MarkdownBlockTreeBuilder.java | 333 ++++++++++++++++ .../CreativeCommonsDocumentTransformer.java | 95 +++++ .../mapper/transform/DocumentTransformer.java | 59 +++ .../transform/DocumentTransformers.java | 74 ++++ .../transform/EpubDocumentTransformer.java | 60 +++ .../transform/FormatCategoryDetector.java | 100 +++++ .../transform/GenericDocumentTransformer.java | 50 +++ .../transform/HtmlDocumentTransformer.java | 65 +++ .../transform/ImageDocumentTransformer.java | 64 +++ .../grpc/mapper/transform/MetadataTagger.java | 116 ++++++ .../transform/OfficeDocumentTransformer.java | 77 ++++ .../transform/PdfDocumentTransformer.java | 65 +++ .../transform/RtfDocumentTransformer.java | 65 +++ .../mapper/transform/TransformSupport.java | 97 +++++ .../transform/WarcDocumentTransformer.java | 58 +++ .../grpc/mapper/ClasspathTestDocuments.java | 111 ++++++ .../tika/grpc/mapper/DocumentBuilderTest.java | 117 ++++++ .../tika/grpc/mapper/ParseFixtureSupport.java | 60 +++ .../content/MarkdownBlockTreeBuilderTest.java | 115 ++++++ ...reativeCommonsDocumentTransformerTest.java | 65 +++ .../transform/DocumentTransformersTest.java | 86 ++++ .../EpubDocumentTransformerTest.java | 57 +++ .../HtmlDocumentTransformerTest.java | 57 +++ .../ImageDocumentTransformerTest.java | 63 +++ .../OfficeDocumentTransformerTest.java | 65 +++ .../transform/PdfDocumentTransformerTest.java | 55 +++ .../transform/RtfDocumentTransformerTest.java | 63 +++ .../WarcDocumentTransformerTest.java | 53 +++ .../test/resources/test-documents/sample.txt | 1 + tika-grpc/README.md | 61 ++- tika-grpc/pom.xml | 13 + .../tika/pipes/grpc/TikaGrpcServerImpl.java | 27 +- tika-grpc/src/main/proto/tika.proto | 12 +- ...BiDirectionalStreamingIntegrationTest.java | 7 +- .../tika/pipes/grpc/TikaGrpcServerTest.java | 3 +- 47 files changed, 3909 insertions(+), 21 deletions(-) create mode 100644 tika-grpc-api/README.md create mode 100644 tika-grpc-api/buf.yaml create mode 100644 tika-grpc-api/pom.xml create mode 100644 tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto create mode 100644 tika-grpc-mapper/docs/EXTENSIONS.md create mode 100644 tika-grpc-mapper/docs/SOURCE_DESTINATION_MAPPING.md create mode 100644 tika-grpc-mapper/docs/TIKA_INTERFACE_MAPPING.md create mode 100644 tika-grpc-mapper/pom.xml create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilder.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformer.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformer.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformers.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/FormatCategoryDetector.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java create mode 100644 tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformer.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ClasspathTestDocuments.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilderTest.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformerTest.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/DocumentTransformersTest.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformerTest.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformerTest.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformerTest.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformerTest.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformerTest.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformerTest.java create mode 100644 tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformerTest.java create mode 100644 tika-grpc-mapper/src/test/resources/test-documents/sample.txt diff --git a/pom.xml b/pom.xml index d0a9958c8d..ffde688d56 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,8 @@ tika-langdetect tika-pipes + tika-grpc-api + tika-grpc-mapper tika-grpc tika-app tika-server diff --git a/tika-bom/pom.xml b/tika-bom/pom.xml index 3f8a473509..8b008ff476 100644 --- a/tika-bom/pom.xml +++ b/tika-bom/pom.xml @@ -447,6 +447,21 @@ tika-async-cli ${revision} + + org.apache.tika + tika-grpc-api + ${revision} + + + org.apache.tika + tika-grpc-mapper + ${revision} + + + org.apache.tika + tika-grpc + ${revision} + org.apache.tika tika-httpclient-commons diff --git a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java index f8e4799331..18ca0c4178 100644 --- a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java +++ b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java @@ -313,7 +313,7 @@ void testParseContextJson() throws Exception { Assertions.assertEquals("PARSE_SUCCESS", htmlReply.getStatus(), "Parse should succeed with HTML handler type"); - String htmlContent = htmlReply.getFieldsMap().get("X-TIKA:content"); + String htmlContent = htmlReply.getDocument().getMarkdown(); Assertions.assertNotNull(htmlContent, "Content should be present in HTML response"); log.info("HTML content (first 200 chars): {}", htmlContent.substring(0, Math.min(200, htmlContent.length()))); Assertions.assertTrue( @@ -331,7 +331,7 @@ void testParseContextJson() throws Exception { Assertions.assertEquals("PARSE_SUCCESS", textReply.getStatus(), "Parse should succeed with TEXT handler type"); - String textContent = textReply.getFieldsMap().get("X-TIKA:content"); + String textContent = textReply.getDocument().getMarkdown(); Assertions.assertNotNull(textContent, "Content should be present in text response"); log.info("Text content (first 200 chars): {}", textContent.substring(0, Math.min(200, textContent.length()))); Assertions.assertFalse( diff --git a/tika-grpc-api/README.md b/tika-grpc-api/README.md new file mode 100644 index 0000000000..3f74dcc9a0 --- /dev/null +++ b/tika-grpc-api/README.md @@ -0,0 +1,55 @@ +# Apache Tika gRPC API + +Typed protobuf messages for Tika parse output under `org.apache.tika.grpc.v1`. + +## Contents + +- **Document** (`document.proto`) — the single, small, stable parse-result contract: a + structured markdown content tree (`blocks`/`markdown`), typed common metadata + (`DocumentMetadata`), and a tagged metadata tail (`extra`) for everything + format-specific. +- **Bundled descriptors** — `META-INF/org.apache.tika.grpc.v1.descriptors` in the + published jar. + +## Usage + +```xml + + org.apache.tika + tika-grpc-api + ${tika.version} + +``` + +## The Document shape + +Rather than one proto message per source format, `Document` models content and +metadata by *concern*, not by *format*: + +- **Content** lives in `markdown` (the authoritative render) and `blocks` (the same + content parsed into a structured tree of headings/paragraphs/lists/tables/code + blocks/inline runs). This is format-agnostic: every Tika parser's output reaches + this shape the same way, via markdown. +- **Metadata** has a small, bounded set of typed common fields on `DocumentMetadata` + (title, authors, description, keywords, languages, dates, counts, dimensions, + rights) plus a tagged tail (`extra`, a `repeated MetadataField`) for everything + format-specific. Tail values are typed where Tika's own `Property` declares a type + (integer/number/boolean/timestamp), and a string otherwise — never guessed. +- **`format_category`** is a cheap routing hint (`FormatCategory` enum: PDF, OFFICE, + IMAGE, HTML, RTF, EPUB, WARC, GENERIC). It is not mutually exclusive with anything + in `extra` — a document can be `FORMAT_CATEGORY_PDF` and still carry Creative + Commons rights metadata, for example. +- **`embedded`** recurses: a PDF with an embedded image is one `Document` whose + `embedded` list contains a fully-typed child `Document` for the image. + +Format-specific mapping (which Tika `Property` becomes which typed field, and what +falls through to `extra`) lives in `tika-grpc-mapper`'s +`org.apache.tika.grpc.mapper.transform.DocumentTransformer` implementations, one per +format — code, not schema. Adding a parser means adding a transformer; the wire +contract does not change. + +## Lint + +```bash +cd tika-grpc-api && buf lint +``` diff --git a/tika-grpc-api/buf.yaml b/tika-grpc-api/buf.yaml new file mode 100644 index 0000000000..69f8cd1593 --- /dev/null +++ b/tika-grpc-api/buf.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: v2 +modules: + - path: src/main/proto + name: buf.build/apache/tika-grpc-api +deps: + - buf.build/googleapis/googleapis +lint: + use: + - STANDARD +breaking: + use: + - FILE diff --git a/tika-grpc-api/pom.xml b/tika-grpc-api/pom.xml new file mode 100644 index 0000000000..175c2cb3b4 --- /dev/null +++ b/tika-grpc-api/pom.xml @@ -0,0 +1,121 @@ + + + + 4.0.0 + tika-grpc-api + jar + Apache Tika gRPC API (typed Document protos) + https://tika.apache.org/ + + + org.apache.tika + tika-parent + ${revision} + ../tika-parent/pom.xml + + + + + com.google.protobuf + protobuf-java + + + + + + + kr.motd.maven + os-maven-plugin + 1.7.1 + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + ${project.basedir}/src/main/proto + + org/apache/tika/grpc/v1/*.proto + + true + true + ${project.build.outputDirectory}/META-INF + org.apache.tika.grpc.v1.descriptors + + + + + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.1 + + + add-protobuf-sources + generate-sources + + add-source + + + + ${project.build.directory}/generated-sources/protobuf/java + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.tika.grpc.api + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + org.apache.rat + apache-rat-plugin + + + README.md + + + + + + diff --git a/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto b/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto new file mode 100644 index 0000000000..e05752b73c --- /dev/null +++ b/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto @@ -0,0 +1,208 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The Document shape: a single, small, stable contract for a parsed Tika document, +// instead of a proto message per source format. +// +// Content model: a structured markdown document tree (blocks + inline runs, with +// tables / task lists / strikethrough). It is anchored to a stable, standard markdown +// shape, so it does not churn, it renders straight back to markdown, and every +// RAG/LLM tool already speaks it. +// +// Metadata: typed common fields (by concern, not by source format) plus a TAGGED tail. +// Tags come from Tika's declared Property value type; unknown keys are strings, never +// guessed. +// +// Format specifics arrive via per-parser DocumentTransformer code, not a proto-per-type, +// so adding a parser never touches the wire. + +syntax = "proto3"; + +package org.apache.tika.grpc.v1; + +import "google/protobuf/timestamp.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.tika.grpc.v1"; + +// ========================================================================= +// Top level +// ========================================================================= +message Document { + // envelope + string id = 1; + string content_type = 2; // Tika-detected MIME type + FormatCategory format_category = 3; // coarse routing hint; not mutually exclusive with `extra` + google.protobuf.Timestamp parsed_at = 4; + ParseStatus status = 5; + SourceOrigin origin = 6; + + // content (structured markdown tree). `markdown` is the authoritative render; + // `blocks` is the same content parsed into a structured tree for programmatic use. + string markdown = 10; + repeated Block blocks = 11; + + // metadata + DocumentMetadata metadata = 20; // typed common fields + repeated MetadataField extra = 21; // tagged tail: everything a transformer didn't type + + // embedded children, recursive (parent PDF + child image, each fully typed) + repeated Document embedded = 30; +} + +// Coarse format category for routing. Cross-cutting concerns (e.g. Creative Commons +// licensing) are NOT categories here -- they surface via `extra`/DocumentMetadata +// regardless of the primary category, since they can coexist with any format. +enum FormatCategory { + FORMAT_CATEGORY_UNSPECIFIED = 0; + FORMAT_CATEGORY_PDF = 1; + FORMAT_CATEGORY_OFFICE = 2; + FORMAT_CATEGORY_IMAGE = 3; + FORMAT_CATEGORY_HTML = 4; + FORMAT_CATEGORY_RTF = 5; + FORMAT_CATEGORY_EPUB = 6; + FORMAT_CATEGORY_WARC = 7; + FORMAT_CATEGORY_GENERIC = 8; +} + +message SourceOrigin { + string filename = 1; + string uri = 2; + int64 byte_size = 3; + string parser = 4; +} + +// ========================================================================= +// Blocks (markdown block structure) +// ========================================================================= +message Block { + oneof block { + Heading heading = 1; // ATX/setext, level 1..6 + Paragraph paragraph = 2; + BlockQuote block_quote = 3; + BulletList bullet_list = 4; + OrderedList ordered_list = 5; + CodeBlock code_block = 6; // fenced or indented + HtmlBlock html_block = 7; + ThematicBreak thematic_break = 8; + Table table = 9; // GFM + } +} + +message Heading { int32 level = 1; repeated Inline content = 2; } +message Paragraph { repeated Inline content = 1; } +message BlockQuote { repeated Block blocks = 1; } +message ThematicBreak {} +message HtmlBlock { string literal = 1; } + +message BulletList { string marker = 1; bool tight = 2; repeated ListItem items = 3; } // marker: - + * +message OrderedList { int32 start = 1; string delimiter = 2; bool tight = 3; repeated ListItem items = 4; } // delimiter: . ) +message ListItem { + TaskStatus task = 1; // GFM task list checkbox + repeated Block blocks = 2; +} +enum TaskStatus { TASK_NONE = 0; TASK_UNCHECKED = 1; TASK_CHECKED = 2; } + +message CodeBlock { bool fenced = 1; string info = 2; string literal = 3; } // info = fence language/info + +message Table { repeated TableCell header = 1; repeated TableRow rows = 2; } +message TableRow { repeated TableCell cells = 1; } +message TableCell { Alignment alignment = 1; repeated Inline content = 2; } +enum Alignment { ALIGN_NONE = 0; ALIGN_LEFT = 1; ALIGN_CENTER = 2; ALIGN_RIGHT = 3; } + +// ========================================================================= +// Inline runs (structured markdown formatting) +// ========================================================================= +message Inline { + oneof inline { + string text = 1; // plain run + Emphasis emphasis = 2; // *italic* + Strong strong = 3; // **bold** + Strikethrough strikethrough = 4; // ~~del~~ (GFM) + Code code = 5; // `code` + Link link = 6; + Image image = 7; + string html = 8; // raw inline html + LineBreak line_break = 9; + } +} +message Emphasis { repeated Inline content = 1; } +message Strong { repeated Inline content = 1; } +message Strikethrough { repeated Inline content = 1; } +message Code { string literal = 1; } +message Link { string destination = 1; string title = 2; repeated Inline content = 3; } +message LineBreak { bool hard = 1; } + +message Image { + string destination = 1; + string title = 2; + string alt = 3; +} + +// ========================================================================= +// Metadata: typed common fields (grouped by concern, not by source format) +// ========================================================================= +message DocumentMetadata { + string title = 1; + repeated string authors = 2; + string description = 3; + repeated string keywords = 4; + repeated string languages = 5; + + google.protobuf.Timestamp created = 10; + google.protobuf.Timestamp modified = 11; + + int32 page_count = 20; + int64 word_count = 21; + int64 character_count = 22; + + string rights = 30; // free-text rights/license statement + string license_url = 31; // e.g. a Creative Commons license URI + + int32 width = 40; + int32 height = 41; + double duration_seconds = 42; + + // Extend by concern (fonts, geo, security) as new common needs appear. This is + // where the "few hundred fields" would live if we chased them -- bounded, + // common, and cross-format. Anything narrower belongs in `extra`. +} + +// ========================================================================= +// Tagged tail: typed where Tika declares the type, string otherwise (never guessed) +// ========================================================================= +message MetadataField { string key = 1; MetadataValue value = 2; } +message MetadataValue { + oneof value { + StringList strings = 1; // default for untyped / multivalue text + int64 integer = 2; + double number = 3; + bool boolean = 4; + google.protobuf.Timestamp timestamp = 5; + } +} +message StringList { repeated string values = 1; } + +message ParseStatus { + enum Status { UNSPECIFIED = 0; SUCCESS = 1; PARTIAL = 2; FAILED = 3; } + Status status = 1; + int64 parse_time_ms = 2; + string parser_used = 3; + repeated string parsers_used = 4; + repeated string warnings = 5; + repeated string errors = 6; +} diff --git a/tika-grpc-mapper/docs/EXTENSIONS.md b/tika-grpc-mapper/docs/EXTENSIONS.md new file mode 100644 index 0000000000..98090d3f8d --- /dev/null +++ b/tika-grpc-mapper/docs/EXTENSIONS.md @@ -0,0 +1,43 @@ +# Document extensions + +Core mapping lives in `org.apache.tika.grpc.mapper.DocumentBuilder`, which delegates +metadata mapping to `org.apache.tika.grpc.mapper.transform.DocumentTransformer` +implementations (one per format concern) and content-tree building to +`org.apache.tika.grpc.mapper.content.MarkdownBlockTreeBuilder` (format-agnostic, since +every Tika parser's output reaches this module as markdown). + +## Adding a new format + +Adding support for a new format means adding a `DocumentTransformer`, not touching the +wire contract: + +1. Implement `DocumentTransformer`: `appliesTo(Metadata)` decides whether this + transformer applies (usually a `Metadata.CONTENT_TYPE` check; cross-cutting + transformers like `CreativeCommonsDocumentTransformer` may inspect the full + metadata instead); `transform(Metadata, Document.Builder)` maps the common, + cross-format facts into `DocumentMetadata` via `TransformSupport`, then calls + `MetadataTagger.appendTail(...)` so every remaining Tika key lands in the tagged + tail, typed where Tika declares the type and string otherwise. +2. Register it in `DocumentTransformers.defaults()`. +3. Add a test that parses a real fixture (via the module's existing + `tika-parser-*-module` test-jar dependencies) and asserts on the fields that + actually populate. + +`PdfDocumentTransformer` is the reference implementation for this pattern. + +## Document outlines / structure + +PDF bookmarks, HTML/Markdown heading trees, and section boundaries are largely already +captured by `Document.blocks` (a `Heading` block carries its `level`, so a heading tree +is just a filter over `blocks`). A dedicated outline extraction pass (e.g. PDF +bookmarks, which are metadata rather than content) is not yet implemented; it would be +a `DocumentTransformer` that populates `DocumentMetadata` or the tagged tail from +PDFBox's outline API, not a proto change. + +## Pluggable, format-unknown output + +For output that does not fit the typed shape at all (e.g. a document-layout model's +own tree), the plan is a pluggable external-parser mechanism carrying opaque +`google.protobuf.Any` payloads on the `Document` — a follow-up change, so that this +contract never has to model a plugin's output shape. Do not add per-format proto +messages here. diff --git a/tika-grpc-mapper/docs/SOURCE_DESTINATION_MAPPING.md b/tika-grpc-mapper/docs/SOURCE_DESTINATION_MAPPING.md new file mode 100644 index 0000000000..058b9e4cfc --- /dev/null +++ b/tika-grpc-mapper/docs/SOURCE_DESTINATION_MAPPING.md @@ -0,0 +1,369 @@ +# Source to Destination Field Mapping + +## 🎯 Purpose +This document provides **exact** mappings from Tika interface properties to our protobuf fields. Use this as the definitive reference when implementing metadata builders. + +--- + +## 📋 PDF Documents + +### Source: `org.apache.tika.metadata.PDF.java` +### Destination: `io.pipeline.parsed.data.pdf.v1.PdfMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `PDF.DOC_INFO_TITLE` | `doc_info_title` | string | PDF document title | +| `PDF.DOC_INFO_AUTHOR` | `doc_info_author` | string | PDF document author | +| `PDF.DOC_INFO_SUBJECT` | `doc_info_subject` | string | PDF document subject | +| `PDF.DOC_INFO_KEYWORDS` | `doc_info_keywords` | string | PDF document keywords | +| `PDF.DOC_INFO_CREATOR` | `doc_info_creator` | string | PDF document creator | +| `PDF.DOC_INFO_PRODUCER` | `doc_info_producer` | string | PDF document producer | +| `PDF.DOC_INFO_CREATED` | `doc_info_created` | Timestamp | PDF creation date | +| `PDF.DOC_INFO_MODIFICATION_DATE` | `doc_info_modification_date` | Timestamp | PDF modification date | +| `PDF.DOC_INFO_TRAPPED` | `doc_info_trapped` | string | PDF trapped status | +| `PDF.PDF_VERSION` | `pdf_version` | string | PDF version | +| `PDF.PDFA_VERSION` | `pdfa_version` | string | PDF/A version | +| `PDF.PDF_EXTENSION_VERSION` | `pdf_extension_version` | string | PDF extension version | +| `PDF.PDFAID_CONFORMANCE` | `pdfaid_conformance` | string | PDF/A ID conformance | +| `PDF.PDFAID_PART` | `pdfaid_part` | int32 | PDF/A ID part | +| `PDF.PDFUAID_PART` | `pdfuaid_part` | int32 | PDF/UA ID part | +| `PDF.PDFVT_VERSION` | `pdfvt_version` | string | PDF/VT version | +| `PDF.PDFVT_MODIFIED` | `pdfvt_modified` | Timestamp | PDF/VT modified date | +| `PDF.PDFXID_VERSION` | `pdfxid_version` | string | PDF/X ID version | +| `PDF.PDFX_VERSION` | `pdfx_version` | string | PDF/X version | +| `PDF.PDFX_CONFORMANCE` | `pdfx_conformance` | string | PDF/X conformance | +| `PDF.IS_ENCRYPTED` | `is_encrypted` | bool | PDF encryption status | +| `PDF.PRODUCER` | `producer` | string | PDF producer | +| `PDF.HAS_XFA` | `has_xfa` | bool | Has XFA forms | +| `PDF.HAS_XMP` | `has_xmp` | bool | Has XMP metadata | +| `PDF.XMP_LOCATION` | `xmp_location` | string | XMP metadata location | +| `PDF.HAS_ACROFORM_FIELDS` | `has_acroform_fields` | bool | Has AcroForm fields | +| `PDF.HAS_MARKED_CONTENT` | `has_marked_content` | bool | Has marked content | +| `PDF.HAS_COLLECTION` | `has_collection` | bool | Has collection (portfolio) | +| `PDF.HAS_3D` | `has_3d` | bool | Has 3D annotations | +| `PDF.NUM_3D_ANNOTATIONS` | `num_3d_annotations` | int32 | Number of 3D annotations | +| `PDF.ACTION_TRIGGER` | `action_trigger` | string | Action trigger location | +| `PDF.ACTION_TRIGGERS` | `action_triggers` | repeated string | All action triggers | +| `PDF.ACTION_TYPES` | `action_types` | repeated string | Action types | +| `PDF.CHARACTERS_PER_PAGE` | `characters_per_page` | repeated int32 | Characters per page | +| `PDF.UNMAPPED_UNICODE_CHARS_PER_PAGE` | `unmapped_unicode_chars_per_page` | repeated int32 | Unmapped unicode chars per page | +| `PDF.TOTAL_UNMAPPED_UNICODE_CHARS` | `total_unmapped_unicode_chars` | int32 | Total unmapped unicode chars | +| `PDF.OVERALL_PERCENTAGE_UNMAPPED_UNICODE_CHARS` | `overall_percentage_unmapped_unicode_chars` | double | Percentage unmapped unicode | +| `PDF.CONTAINS_DAMAGED_FONT` | `contains_damaged_font` | bool | Contains damaged fonts | +| `PDF.CONTAINS_NON_EMBEDDED_FONT` | `contains_non_embedded_font` | bool | Contains non-embedded fonts | +| `PDF.EMBEDDED_FILE_DESCRIPTION` | `embedded_file_description` | string | Embedded file description | +| `PDF.EMBEDDED_FILE_ANNOTATION_TYPE` | `embedded_file_annotation_type` | string | Embedded file annotation type | +| `PDF.EMBEDDED_FILE_SUBTYPE` | `embedded_file_subtype` | string | Embedded file subtype | +| `PDF.ANNOTATION_TYPES` | `annotation_types` | repeated string | Annotation types | +| `PDF.ANNOTATION_SUBTYPES` | `annotation_subtypes` | repeated string | Annotation subtypes | +| `PDF.ASSOCIATED_FILE_RELATIONSHIP` | `associated_file_relationship` | string | Associated file relationship | +| `PDF.INCREMENTAL_UPDATE_NUMBER` | `incremental_update_number` | int32 | Incremental update number | +| `PDF.PDF_INCREMENTAL_UPDATE_COUNT` | `pdf_incremental_update_count` | int32 | Incremental update count | +| `PDF.OCR_PAGE_COUNT` | `ocr_page_count` | int32 | OCR page count | +| `PDF.EOF_OFFSETS` | `eof_offsets` | repeated double | EOF offsets | + +### Source: `org.apache.tika.metadata.XMPPDF.java` +### Destination: `io.pipeline.parsed.data.pdf.v1.PdfMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `XMPPDF.KEY_WORDS` | `xmp_keywords` | string | XMP PDF keywords (constant is KEY_WORDS in Tika 2.x snapshot) | +| `XMPPDF.PDF_VERSION` | `pdf_version` | string | XMP PDF version (normalized to `pdf_version`) | +| `XMPPDF.PRODUCER` | `producer` | string | XMP PDF producer (normalized to `producer`) | + +### Source: `org.apache.tika.metadata.AccessPermissions.java` +### Destination: `io.pipeline.parsed.data.pdf.v1.PdfMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `AccessPermissions.ASSEMBLE_DOCUMENT` | `can_assemble_document` | bool | Can assemble document | +| `AccessPermissions.EXTRACT_CONTENT` | `can_extract_content` | bool | Can extract content | +| `AccessPermissions.EXTRACT_FOR_ACCESSIBILITY` | `can_extract_for_accessibility` | bool | Can extract for accessibility | +| `AccessPermissions.FILL_IN_FORM` | `can_fill_in_form` | bool | Can fill in forms | +| `AccessPermissions.CAN_MODIFY_ANNOTATIONS` | `can_modify_annotations` | bool | Can modify annotations | +| `AccessPermissions.CAN_MODIFY` | `can_modify_document` | bool | Can modify document | +| `AccessPermissions.CAN_PRINT` | `can_print` | bool | Can print | +| `AccessPermissions.CAN_PRINT_FAITHFUL` | `can_print_faithful` | bool | Can print faithful | + +Notes: +- We normalize to expressive `can_*` boolean fields and align names/types with Tika snapshot. +### Additional PDF-related fields +### Sources: `org.apache.tika.metadata.PagedText.java`, PDF parser literals +### Destination: `io.pipeline.parsed.data.pdf.v1.PdfMetadata` + +| Source | Protobuf Field | Type | Notes | +|--------|----------------|------|-------| +| `PagedText.N_PAGES` | `n_pages` | int32 | Total number of pages | +| `"X-TIKA:pdf:metadata-xmp-parse-failed"` | `xmp_parse_failed` | repeated string | XMP parse failure messages | +| `PDF.ILLUSTRATOR_TYPE` | `illustrator_type` | string | Illustrator type if present | +| `"pdf:foundNonAdobeExtensionName"` | — | — | Kept in `ParseResponse.metadata` mirror (literal key) | + +--- + +## 📋 Office Documents + +### Source: `org.apache.tika.metadata.Office.java` +### Destination: `io.pipeline.parsed.data.office.v1.OfficeMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `Office.CHARACTER_COUNT` | `character_count` | int32 | Character count | +| `Office.CHARACTER_COUNT_WITH_SPACES` | `character_count_with_spaces` | int32 | Character count with spaces | +| `Office.LINE_COUNT` | `line_count` | int32 | Line count | +| `Office.PAGE_COUNT` | `page_count` | int32 | Page count | +| `Office.PARAGRAPH_COUNT` | `paragraph_count` | int32 | Paragraph count | +| `Office.WORD_COUNT` | `word_count` | int32 | Word count | +| `Office.SLIDE_COUNT` | `slide_count` | int32 | Slide count | +| `Office.NOTES_COUNT` | `notes_count` | int32 | Notes count | +| `Office.HIDDEN_COUNT` | `hidden_count` | int32 | Hidden slide count | +| `Office.MULTIMEDIA_OBJECT_COUNT` | `multimedia_object_count` | int32 | Multimedia object count | + +### Source: `org.apache.tika.metadata.OfficeOpenXMLCore.java` +### Destination: `io.pipeline.parsed.data.office.v1.OfficeMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `OfficeOpenXMLCore.CATEGORY` | `category` | string | Document category | +| `OfficeOpenXMLCore.CONTENT_STATUS` | `content_status` | string | Content status | +| `OfficeOpenXMLCore.CREATED` | `created` | Timestamp | Creation date | +| `OfficeOpenXMLCore.CREATOR` | `creator` | string | Creator | +| `OfficeOpenXMLCore.DESCRIPTION` | `description` | string | Description | +| `OfficeOpenXMLCore.IDENTIFIER` | `identifier` | string | Identifier | +| `OfficeOpenXMLCore.KEYWORDS` | `keywords` | string | Keywords | +| `OfficeOpenXMLCore.LANGUAGE` | `language` | string | Language | +| `OfficeOpenXMLCore.LAST_MODIFIED_BY` | `last_modified_by` | string | Last modified by | +| `OfficeOpenXMLCore.LAST_PRINTED` | `last_printed` | Timestamp | Last printed | +| `OfficeOpenXMLCore.MODIFIED` | `modified` | Timestamp | Modified date | +| `OfficeOpenXMLCore.REVISION` | `revision` | string | Revision | +| `OfficeOpenXMLCore.SUBJECT` | `subject` | string | Subject | +| `OfficeOpenXMLCore.TITLE` | `title` | string | Title | +| `OfficeOpenXMLCore.VERSION` | `version` | string | Version | + +### Source: `org.apache.tika.metadata.OfficeOpenXMLExtended.java` +### Destination: `io.pipeline.parsed.data.office.v1.OfficeMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `OfficeOpenXMLExtended.APPLICATION` | `application` | string | Application name | +| `OfficeOpenXMLExtended.APPLICATION_VERSION` | `application_version` | string | Application version | +| `OfficeOpenXMLExtended.CHARACTERS` | `characters` | int64 | Character count | +| `OfficeOpenXMLExtended.CHARACTERS_WITH_SPACES` | `characters_with_spaces` | int64 | Characters with spaces | +| `OfficeOpenXMLExtended.COMPANY` | `company` | string | Company | +| `OfficeOpenXMLExtended.DOC_SECURITY` | `doc_security` | int32 | Document security | +| `OfficeOpenXMLExtended.HIDDEN_SLIDES` | `hidden_slides` | int32 | Hidden slides | +| `OfficeOpenXMLExtended.LINES` | `lines` | int32 | Lines | +| `OfficeOpenXMLExtended.MANAGER` | `manager` | string | Manager | +| `OfficeOpenXMLExtended.MULTIMEDIA_CLIPS` | `multimedia_clips` | int32 | Multimedia clips | +| `OfficeOpenXMLExtended.NOTES` | `notes` | int32 | Notes | +| `OfficeOpenXMLExtended.PAGES` | `pages` | int32 | Pages | +| `OfficeOpenXMLExtended.PARAGRAPHS` | `paragraphs` | int32 | Paragraphs | +| `OfficeOpenXMLExtended.PRESENTATION_FORMAT` | `presentation_format` | string | Presentation format | +| `OfficeOpenXMLExtended.SLIDES` | `slides` | int32 | Slides | +| `OfficeOpenXMLExtended.TEMPLATE` | `template` | string | Template | +| `OfficeOpenXMLExtended.TOTAL_TIME` | `total_time_raw` / `total_time_seconds` | string / int32 | Raw duration and parsed seconds | +| `OfficeOpenXMLExtended.WORDS` | `words` | int32 | Words | + +--- + +## 📋 Image Documents + +### Source: `org.apache.tika.metadata.TIFF.java` +### Destination: `io.pipeline.parsed.data.image.v1.ImageMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `TIFF.IMAGE_WIDTH` | `image_width` | int32 | Image width | +| `TIFF.IMAGE_LENGTH` | `image_length` | int32 | Image height | +| `TIFF.BITS_PER_SAMPLE` | `bits_per_sample` | repeated int32 | Bits per sample | +| `TIFF.COMPRESSION` | `compression` | int32 | Compression type | +| `TIFF.PHOTOMETRIC_INTERPRETATION` | `photometric_interpretation` | int32 | Photometric interpretation | +| `TIFF.SAMPLES_PER_PIXEL` | `samples_per_pixel` | int32 | Samples per pixel | +| `TIFF.PLANAR_CONFIGURATION` | `planar_configuration` | int32 | Planar configuration | +| `TIFF.RESOLUTION_UNIT` | `resolution_unit` | int32 | Resolution unit | +| `TIFF.X_RESOLUTION` | `x_resolution` | double | X resolution | +| `TIFF.Y_RESOLUTION` | `y_resolution` | double | Y resolution | +| `TIFF.ORIENTATION` | `orientation` | int32 | Image orientation | + +### Source: `org.apache.tika.metadata.IPTC.java` +### Destination: `io.pipeline.parsed.data.image.v1.ImageMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `IPTC.KEYWORDS` | `iptc_keywords` | repeated string | IPTC keywords | +| `IPTC.CAPTION_ABSTRACT` | `iptc_caption` | string | IPTC caption | +| `IPTC.HEADLINE` | `iptc_headline` | string | IPTC headline | +| `IPTC.CREDIT` | `iptc_credit` | string | IPTC credit | +| `IPTC.SOURCE` | `iptc_source` | string | IPTC source | +| `IPTC.COPYRIGHT_NOTICE` | `iptc_copyright` | string | IPTC copyright | +| `IPTC.CATEGORY` | `iptc_category` | string | IPTC category | +| `IPTC.SUPPLEMENTAL_CATEGORIES` | `iptc_supplemental_categories` | repeated string | IPTC supplemental categories | + +--- + +## 📋 Email Documents + +### Source: `org.apache.tika.metadata.Message.java` +### Destination: `io.pipeline.parsed.data.email.v1.EmailMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `Message.MESSAGE_FROM` | `from_address` | string | From address | +| `Message.MESSAGE_TO` | `to_addresses` | repeated string | To addresses | +| `Message.MESSAGE_CC` | `cc_addresses` | repeated string | CC addresses | +| `Message.MESSAGE_BCC` | `bcc_addresses` | repeated string | BCC addresses | +| `Message.MESSAGE_SUBJECT` | `subject` | string | Email subject | +| `Message.MESSAGE_DATE` | `message_date` | Timestamp | Message date | +| `Message.MESSAGE_ID` | `message_id` | string | Message ID | +| `Message.MESSAGE_IN_REPLY_TO` | `in_reply_to` | string | In reply to | +| `Message.MESSAGE_REFERENCES` | `references` | repeated string | References | + +### Source: `org.apache.tika.metadata.MAPI.java` +### Destination: `io.pipeline.parsed.data.email.v1.EmailMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `MAPI.MESSAGE_CLASS` | `mapi_message_class` | string | MAPI message class | +| `MAPI.CONVERSATION_TOPIC` | `mapi_conversation_topic` | string | MAPI conversation topic | + +--- + +## 📋 Media Documents + +### Source: `org.apache.tika.metadata.XMPDM.java` +### Destination: `io.pipeline.parsed.data.media.v1.MediaMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `XMPDM.DURATION` | `duration_seconds` | double | Duration in seconds | +| `XMPDM.AUDIO_SAMPLE_RATE` | `audio_sample_rate` | int32 | Audio sample rate | +| `XMPDM.AUDIO_CHANNELS` | `audio_channels` | int32 | Audio channels | +| `XMPDM.VIDEO_FRAME_RATE` | `video_frame_rate` | double | Video frame rate | +| `XMPDM.VIDEO_WIDTH` | `video_width` | int32 | Video width | +| `XMPDM.VIDEO_HEIGHT` | `video_height` | int32 | Video height | +| `XMPDM.ARTIST` | `artist` | string | Artist | +| `XMPDM.ALBUM` | `album` | string | Album | +| `XMPDM.TRACK_NUMBER` | `track_number` | int32 | Track number | +| `XMPDM.GENRE` | `genre` | string | Genre | +| `XMPDM.COMPOSER` | `composer` | string | Composer | + +--- + +## 📋 HTML Documents + +### Source: `org.apache.tika.metadata.HTML.java` +### Destination: `io.pipeline.parsed.data.html.v1.HtmlMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `HTML.DESCRIPTION` | `description` | string | HTML description | +| `HTML.KEYWORDS` | `keywords` | repeated string | HTML keywords | +| `HTML.REFRESH` | `refresh` | string | HTML refresh | + +--- + +## 📋 Creative Commons Documents + +### Source: `org.apache.tika.metadata.CreativeCommons.java` +### Destination: `io.pipeline.parsed.data.creative_commons.v1.CreativeCommonsMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `CreativeCommons.LICENSE_URL` | `license_url` | string | License URL | +| `CreativeCommons.LICENSE_LOCATION` | `license_location` | string | License location | +| `CreativeCommons.WORK_TYPE` | `work_type` | string | Work type | + +### Source: `org.apache.tika.metadata.XMPRights.java` +### Destination: `io.pipeline.parsed.data.creative_commons.v1.CreativeCommonsMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `XMPRights.CERTIFICATE` | `rights_certificate` | string | Rights certificate | +| `XMPRights.MARKED` | `rights_marked` | bool | Rights marked | +| `XMPRights.OWNER` | `rights_owners` | repeated string | Rights owners | +| `XMPRights.USAGE_TERMS` | `usage_terms` | string | Usage terms | +| `XMPRights.WEB_STATEMENT` | `web_statement` | string | Web statement | + +--- + +## 📋 Font Documents + +### Source: `org.apache.tika.metadata.Font.java` +### Destination: `io.pipeline.parsed.data.tika.font.v1.FontMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `Font.FONT_NAME` | `font_name` | string | Font name | + +--- + +## 📋 EPUB Documents + +### Source: `org.apache.tika.metadata.Epub.java` +### Destination: `io.pipeline.parsed.data.epub.v1.EpubMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `Epub.RENDITION_LAYOUT` | `rendition_layout` | string | Rendition layout | +| `Epub.VERSION` | `version` | string | EPUB version | + +--- + +## 📋 WARC Documents + +### Source: `org.apache.tika.metadata.WARC.java` +### Destination: `io.pipeline.parsed.data.warc.v1.WarcMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `WARC.WARC_RECORD_ID` | `warc_record_id` | string | WARC record ID | +| `WARC.WARC_RECORD_CONTENT_TYPE` | `warc_record_content_type` | string | WARC record content type | +| `WARC.WARC_RECORD_DATE` | `warc_record_date` | Timestamp | WARC record date | +| `WARC.WARC_RECORD_TYPE` | `warc_record_type` | string | WARC record type | + +--- + +## 📋 Climate Forecast Documents + +### Source: `org.apache.tika.metadata.ClimateForcast.java` +### Destination: `io.pipeline.parsed.data.climate.v1.ClimateForcastMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `ClimateForcast.PROGRAM_ID` | `program_id` | string | Program ID | +| `ClimateForcast.INSTITUTION` | `institution` | string | Institution | +| `ClimateForcast.EXPERIMENT_ID` | `experiment_id` | string | Experiment ID | +| `ClimateForcast.MODEL_ID` | `model_id` | string | Model ID | +| `ClimateForcast.REALIZATION` | `realization` | string | Realization | +| `ClimateForcast.FREQUENCY` | `frequency` | string | Frequency | +| `ClimateForcast.REALM` | `realm` | string | Realm | +| `ClimateForcast.VARIABLE_ID` | `variable_id` | string | Variable ID | +| `ClimateForcast.ENSEMBLE` | `ensemble` | string | Ensemble | +| `ClimateForcast.GRID_LABEL` | `grid_label` | string | Grid label | +| `ClimateForcast.NOMINAL_RESOLUTION` | `nominal_resolution` | string | Nominal resolution | +| `ClimateForcast.SOURCE_ID` | `source_id` | string | Source ID | +| `ClimateForcast.SUB_EXPERIMENT_ID` | `sub_experiment_id` | string | Sub experiment ID | +| `ClimateForcast.TABLE_ID` | `table_id` | string | Table ID | +| `ClimateForcast.VARIANT_LABEL` | `variant_label` | string | Variant label | +| `ClimateForcast.ACTIVITY_ID` | `activity_id` | string | Activity ID | + +--- + +## 🔧 Usage Instructions + +### For Implementing Metadata Builders: + +1. **Find your document type** in this mapping +2. **Use the exact Tika Property names** from the "Tika Property" column +3. **Map to the exact protobuf field names** from the "Protobuf Field" column +4. **Use the correct data types** as specified in the "Type" column +5. **Any unmapped fields** are still present in the `ParseResponse.metadata` mirror (every key, typed) + +### Example Implementation: +```java +// For PDF metadata builder +MetadataUtils.mapStringField(tikaMetadata, PDF.DOC_INFO_TITLE, builder::setDocInfoTitle, mappedFields); +MetadataUtils.mapTimestampField(tikaMetadata, PDF.DOC_INFO_CREATED, builder::setDocInfoCreated, mappedFields); +MetadataUtils.mapBooleanField(tikaMetadata, PDF.IS_ENCRYPTED, builder::setIsEncrypted, mappedFields); +``` + +This mapping ensures **exact** correspondence between what Tika extracts and what our protobuf captures! diff --git a/tika-grpc-mapper/docs/TIKA_INTERFACE_MAPPING.md b/tika-grpc-mapper/docs/TIKA_INTERFACE_MAPPING.md new file mode 100644 index 0000000000..2369ef0fc1 --- /dev/null +++ b/tika-grpc-mapper/docs/TIKA_INTERFACE_MAPPING.md @@ -0,0 +1,253 @@ +# Tika Interface to Protobuf Mapping + +## 🎯 Purpose +This document maps **actual** Tika metadata interfaces to our protobuf structures. Every interface listed here has been verified to exist in the Tika source code. + +## 📋 Verified Tika Metadata Interfaces + +### Core Document Interfaces +- ✅ `PDF.java` - PDF document properties +- ✅ `Office.java` - Basic Office document properties +- ✅ `OfficeOpenXMLCore.java` - OOXML core properties +- ✅ `OfficeOpenXMLExtended.java` - OOXML extended properties +- ✅ `Message.java` - Email message properties +- ✅ `HTML.java` - HTML document properties +- ✅ `RTFMetadata.java` - RTF document properties +- ✅ `Database.java` - Database file properties +- ✅ `Font.java` - Font file properties +- ✅ `Epub.java` - EPUB book properties +- ✅ `WARC.java` - Web archive properties +- ✅ `ClimateForcast.java` - NetCDF/Climate data properties +- ✅ `CreativeCommons.java` - Creative Commons licensing + +### Image and Media Interfaces +- ✅ `TIFF.java` - TIFF image properties +- ✅ `IPTC.java` - IPTC image metadata +- ✅ `Photoshop.java` - Photoshop-specific metadata +- ✅ `XMPDM.java` - XMP Digital Media properties + +### XMP Interfaces +- ✅ `XMP.java` - XMP Basic Schema +- ✅ `XMPDC.java` - XMP Dublin Core +- ✅ `XMPMM.java` - XMP Media Management +- ✅ `XMPPDF.java` - XMP PDF properties +- ✅ `XMPRights.java` - XMP Rights Management +- ✅ `XMPIdq.java` - XMP Identifier Qualifier + +### Specialized Interfaces +- ✅ `MAPI.java` - MAPI (Outlook) properties +- ✅ `PST.java` - PST file properties +- ✅ `QuattroPro.java` - QuattroPro spreadsheet +- ✅ `WordPerfect.java` - WordPerfect document +- ✅ `MachineMetadata.java` - Machine/system metadata +- ✅ `ExternalProcess.java` - External process metadata +- ✅ `Geographic.java` - Geographic/location data +- ✅ `AccessPermissions.java` - Document access permissions +- ✅ `FileSystem.java` - File system metadata +- ✅ `HttpHeaders.java` - HTTP header metadata +- ✅ `Rendering.java` - Rendering properties +- ✅ `PagedText.java` - Paged text properties +- ✅ `TikaPagedText.java` - Tika-specific paged text + +### Core Framework +- ✅ `DublinCore.java` - Dublin Core standard +- ✅ `TikaCoreProperties.java` - Core Tika properties +- ✅ `TikaMimeKeys.java` - MIME type keys + +--- + +## 🗺️ Interface to Protobuf Mapping + +### 1. PDF Documents +**Tika Interfaces:** +- `PDF.java` - 50+ properties (DOC_INFO_*, PDF_VERSION, IS_ENCRYPTED, etc.) +- `XMPPDF.java` - XMP PDF-specific properties +- `AccessPermissions.java` - PDF security permissions + +**Protobuf Target:** +- `io.pipeline.parsed.data.pdf.v1.PdfMetadata` + +**Key Properties to Map:** +```java +// From PDF.java +PDF.DOC_INFO_TITLE → title +PDF.DOC_INFO_AUTHOR → author +PDF.DOC_INFO_SUBJECT → subject +PDF.DOC_INFO_KEYWORDS → keywords +PDF.DOC_INFO_CREATOR → creator +PDF.DOC_INFO_PRODUCER → producer +PDF.DOC_INFO_CREATED → creation_date +PDF.DOC_INFO_MODIFICATION_DATE → modification_date +PDF.PDF_VERSION → pdf_version +PDF.PDFA_VERSION → pdfa_version +PDF.IS_ENCRYPTED → is_encrypted +PDF.HAS_XFA → has_xfa +PDF.HAS_ACROFORM_FIELDS → has_acroform_fields +PDF.HAS_MARKED_CONTENT → has_marked_content +PDF.HAS_COLLECTION → has_collection +PDF.HAS_3D → has_3d +// ... and 35+ more actual properties + +// From XMPPDF.java (note: constant is KEY_WORDS in snapshot) +XMPPDF.KEY_WORDS → xmp_keywords (string) + +// From AccessPermissions.java (booleans) +AccessPermissions.CAN_MODIFY → can_modify_document +AccessPermissions.EXTRACT_CONTENT → can_extract_content +AccessPermissions.EXTRACT_FOR_ACCESSIBILITY → can_extract_for_accessibility +AccessPermissions.ASSEMBLE_DOCUMENT → can_assemble_document +AccessPermissions.FILL_IN_FORM → can_fill_in_form +AccessPermissions.CAN_MODIFY_ANNOTATIONS → can_modify_annotations +AccessPermissions.CAN_PRINT → can_print +AccessPermissions.CAN_PRINT_FAITHFUL → can_print_faithful + +// From PagedText.java +PagedText.N_PAGES → n_pages (int32) + +// Literal keys from PDF parsers +"X-TIKA:pdf:metadata-xmp-parse-failed" → xmp_parse_failed (repeated string) +"pdf:foundNonAdobeExtensionName" → kept in the `ParseResponse.metadata` mirror +``` + +### 2. Office Documents +**Tika Interfaces:** +- `Office.java` - Basic Office properties +- `OfficeOpenXMLCore.java` - OOXML core metadata +- `OfficeOpenXMLExtended.java` - OOXML extended metadata + +**Protobuf Target:** +- `io.pipeline.parsed.data.office.v1.OfficeMetadata` + +### 3. Image Documents +**Tika Interfaces:** +- `TIFF.java` - TIFF-specific properties +- `IPTC.java` - IPTC metadata standard +- `Photoshop.java` - Photoshop metadata +- `XMP.java` - XMP basic properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.image.v1.ImageMetadata` + +### 4. Email Documents +**Tika Interfaces:** +- `Message.java` - Email message properties +- `MAPI.java` - Outlook/MAPI properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.email.v1.EmailMetadata` + +### 5. Media Documents +**Tika Interfaces:** +- `XMPDM.java` - XMP Digital Media properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.media.v1.MediaMetadata` + +### 6. HTML Documents +**Tika Interfaces:** +- `HTML.java` - HTML metadata properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.html.v1.HtmlMetadata` + +### 7. RTF Documents +**Tika Interfaces:** +- `RTFMetadata.java` - RTF-specific properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.rtf.v1.RtfMetadata` + +### 8. Database Documents +**Tika Interfaces:** +- `Database.java` - Database file properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.database.v1.DatabaseMetadata` + +### 9. Font Documents +**Tika Interfaces:** +- `Font.java` - Font file properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.tika.font.v1.FontMetadata` + +### 10. EPUB Documents +**Tika Interfaces:** +- `Epub.java` - EPUB book properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.epub.v1.EpubMetadata` + +### 11. WARC Documents +**Tika Interfaces:** +- `WARC.java` - Web archive properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.warc.v1.WarcMetadata` + +### 12. Climate Forecast Documents +**Tika Interfaces:** +- `ClimateForcast.java` - NetCDF/Climate properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.climate.v1.ClimateForcastMetadata` + +### 13. Creative Commons Documents +**Tika Interfaces:** +- `CreativeCommons.java` - CC licensing properties +- `XMPRights.java` - XMP rights management + +**Protobuf Target:** +- `io.pipeline.parsed.data.creative_commons.v1.CreativeCommonsMetadata` + +### 14. Generic Documents +**Tika Interfaces:** +- `PST.java` - PST file properties +- `QuattroPro.java` - QuattroPro spreadsheet +- `WordPerfect.java` - WordPerfect document +- `MachineMetadata.java` - Machine metadata +- `ExternalProcess.java` - External process metadata +- `Geographic.java` - Geographic data +- `FileSystem.java` - File system metadata +- `HttpHeaders.java` - HTTP headers +- `Rendering.java` - Rendering properties +- `PagedText.java` - Paged text properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.generic.v1.GenericMetadata` + +--- + +## 🔧 Implementation Strategy + +### Phase 1: Verify Protobuf Fields Match Tika Interfaces +For each document type, ensure our protobuf has fields that correspond to the **actual** Tika interface properties. + +### Phase 2: Create Accurate Builders +Build metadata extractors that map **only** the properties that actually exist in the Tika interfaces. + +### Phase 3: Handle Unmapped Fields +Every Tika key — mapped or not — is mirrored into `ParseResponse.metadata` as a typed, +multivalue-preserving `MetadataEntry`. This single lossless channel replaces the former +per-format `google.protobuf.Struct` catch-alls. + +--- + +## ⚠️ Important Notes + +1. **Only use properties that actually exist** in the Tika interfaces (or clearly documented literal keys from parser code) +2. **Check property types** - some are `Property.internalText()`, others are `Property.internalInteger()`, etc. +3. **Handle multiple interfaces** - some document types (like Office) have multiple related interfaces +4. **Use proper Property constants** - don't use string literals, use the actual Property constants +5. **Lossless mirror** - every key (mapped or not) also appears in `ParseResponse.metadata` + +--- + +## 🎯 Next Steps + +1. **Verify our protobuf fields** match the actual Tika interface properties. If we intentionally rename for clarity, document it in SOURCE_DESTINATION_MAPPING.md. +2. **Create simple, accurate builders** that map only what exists +3. **Test with real documents** to ensure we capture everything Tika extracts +4. **Follow the principle**: "Whatever Tika extracts, we save - strongly-typed if we recognize it, and always in the `metadata` mirror regardless" + +This mapping ensures we build **accurate** metadata extractors based on what Tika actually provides, not what we assume it should provide. diff --git a/tika-grpc-mapper/pom.xml b/tika-grpc-mapper/pom.xml new file mode 100644 index 0000000000..8abed989f8 --- /dev/null +++ b/tika-grpc-mapper/pom.xml @@ -0,0 +1,213 @@ + + + + 4.0.0 + tika-grpc-mapper + jar + Apache Tika gRPC mapper (Metadata to Document) + https://tika.apache.org/ + + + org.apache.tika + tika-parent + ${revision} + ../tika-parent/pom.xml + + + + + ${project.groupId} + tika-grpc-api + ${project.version} + + + ${project.groupId} + tika-core + ${project.version} + + + com.google.protobuf + protobuf-java + + + org.slf4j + slf4j-api + + + org.commonmark + commonmark + + + org.commonmark + commonmark-ext-gfm-tables + + + org.commonmark + commonmark-ext-gfm-strikethrough + + + org.junit.jupiter + junit-jupiter + test + + + ${project.groupId} + tika-parsers-standard-package + ${project.version} + pom + test + + + ${project.groupId} + tika-core + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-pdf-module + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-html-module + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-mail-module + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-microsoft-module + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-image-module + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-text-module + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-font-module + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-audiovideo-module + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-webarchive-module + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-miscoffice-module + ${project.version} + test-jar + test + + + ${project.groupId} + tika-parser-scientific-module + ${project.version} + test + + + ${project.groupId} + tika-parser-scientific-package + ${project.version} + pom + test + + + ${project.groupId} + tika-parser-scientific-module + ${project.version} + test-jar + test + + + org.apache.logging.log4j + log4j-slf4j2-impl + test + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.tika.grpc.mapper + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + org.apache.rat + apache-rat-plugin + + + docs/** + **/test-documents/** + + + + + + diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java new file mode 100644 index 0000000000..1bbee0045e --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper; + +import java.time.Instant; +import java.util.List; + +import org.apache.tika.grpc.mapper.content.MarkdownBlockTreeBuilder; +import org.apache.tika.grpc.mapper.transform.DocumentTransformers; +import org.apache.tika.grpc.mapper.transform.FormatCategoryDetector; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.ParseStatus; +import org.apache.tika.grpc.v1.SourceOrigin; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * Builds a {@link Document} from Tika's parse output: the pipes default content handler + * renders the extracted content as markdown, so this class parses that markdown into + * the structured {@code Block} tree once (format-agnostic) and delegates metadata + * mapping to {@link DocumentTransformers} (format-specific, one transformer per + * concern). Embedded documents recurse into {@link Document#getEmbeddedList()}. + */ +public final class DocumentBuilder { + + private static final DocumentTransformers TRANSFORMERS = DocumentTransformers.defaults(); + + private DocumentBuilder() { + } + + /** Maps Tika parse output (primary metadata plus optional embedded metadata list) to {@link Document}. */ + public static Document build(Metadata primary, List allMetadata, String markdownBody, + String docId, String pipesStatus, long parseTimeMs) { + if (primary == null) { + return Document.newBuilder() + .setStatus(ParseStatus.newBuilder() + .setStatus(ParseStatus.Status.FAILED) + .setParseTimeMs(parseTimeMs) + .addErrors("No metadata returned from parse") + .build()) + .build(); + } + + Document.Builder document = buildOne(primary, markdownBody, docId); + + String parserClass = primary.get(TikaCoreProperties.TIKA_PARSED_BY); + ParseStatus.Builder statusBuilder = ParseStatus.newBuilder() + .setStatus(mapPipesStatus(pipesStatus)) + .setParseTimeMs(parseTimeMs); + if (parserClass != null && !parserClass.isEmpty()) { + statusBuilder.setParserUsed(parserClass); + } + String parsedByFull = primary.get(TikaCoreProperties.TIKA_PARSED_BY_FULL_SET); + if (parsedByFull != null && !parsedByFull.isEmpty()) { + for (String p : parsedByFull.split(",")) { + if (!p.trim().isEmpty()) { + statusBuilder.addParsersUsed(p.trim()); + } + } + } + document.setStatus(statusBuilder.build()); + + if (allMetadata != null && allMetadata.size() > 1) { + for (int i = 1; i < allMetadata.size(); i++) { + Metadata embedded = allMetadata.get(i); + String embeddedBody = embedded.get(TikaCoreProperties.TIKA_CONTENT); + document.addEmbedded(buildOne(embedded, embeddedBody, docId + "#" + i).build()); + } + } + + return document.build(); + } + + private static Document.Builder buildOne(Metadata tika, String markdownBody, String docId) { + Document.Builder document = Document.newBuilder(); + if (docId != null && !docId.isEmpty()) { + document.setId(docId); + } + + String contentType = tika.get(Metadata.CONTENT_TYPE); + if (contentType != null && !contentType.isBlank()) { + document.setContentType(contentType.trim()); + } + document.setFormatCategory(FormatCategoryDetector.detect(tika)); + + Instant now = Instant.now(); + document.setParsedAt(com.google.protobuf.Timestamp.newBuilder() + .setSeconds(now.getEpochSecond()) + .setNanos(now.getNano()) + .build()); + + SourceOrigin.Builder origin = SourceOrigin.newBuilder(); + String resourceName = tika.get(TikaCoreProperties.RESOURCE_NAME_KEY); + if (resourceName != null && !resourceName.isBlank()) { + origin.setFilename(resourceName.trim()); + } + String parserClass = tika.get(TikaCoreProperties.TIKA_PARSED_BY); + if (parserClass != null && !parserClass.isBlank()) { + origin.setParser(parserClass.trim()); + } + document.setOrigin(origin.build()); + + if (markdownBody != null && !markdownBody.isEmpty()) { + document.setMarkdown(markdownBody); + document.addAllBlocks(MarkdownBlockTreeBuilder.toBlocks(markdownBody)); + } + + TRANSFORMERS.transform(tika, document); + + return document; + } + + /** + * Maps {@code org.apache.tika.pipes.api.PipesResult.RESULT_STATUS.name()} to + * {@link ParseStatus.Status}. Values are named after the real enum constants (not + * guessed) since this module intentionally has no compile dependency on tika-pipes-api: + * a clean success (e.g. {@code PARSE_SUCCESS}, {@code EMIT_SUCCESS}) is {@code SUCCESS}; + * a success that happened alongside a caught exception (e.g. + * {@code PARSE_SUCCESS_WITH_EXCEPTION}) is {@code PARTIAL}; everything else -- including + * process crashes like {@code TIMEOUT} and {@code OOM}, which are not partial successes -- + * is {@code FAILED}. + */ + private static ParseStatus.Status mapPipesStatus(String pipesStatus) { + if (pipesStatus == null || pipesStatus.isEmpty()) { + return ParseStatus.Status.UNSPECIFIED; + } + return switch (pipesStatus) { + case "EMPTY_OUTPUT", "PARSE_SUCCESS", "EMIT_SUCCESS", "EMIT_SUCCESS_PASSBACK" -> + ParseStatus.Status.SUCCESS; + case "PARSE_SUCCESS_WITH_EXCEPTION", "PARSE_EXCEPTION_NO_EMIT", "EMIT_SUCCESS_PARSE_EXCEPTION" -> + ParseStatus.Status.PARTIAL; + default -> ParseStatus.Status.FAILED; + }; + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilder.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilder.java new file mode 100644 index 0000000000..fadbb3be7d --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilder.java @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.content; + +import java.util.ArrayList; +import java.util.List; + +import org.commonmark.Extension; +import org.commonmark.ext.gfm.strikethrough.Strikethrough; +import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension; +import org.commonmark.ext.gfm.tables.TableBlock; +import org.commonmark.ext.gfm.tables.TableBody; +import org.commonmark.ext.gfm.tables.TableCell; +import org.commonmark.ext.gfm.tables.TableHead; +import org.commonmark.ext.gfm.tables.TableRow; +import org.commonmark.ext.gfm.tables.TablesExtension; +import org.commonmark.node.BlockQuote; +import org.commonmark.node.BulletList; +import org.commonmark.node.Code; +import org.commonmark.node.Emphasis; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Heading; +import org.commonmark.node.HtmlBlock; +import org.commonmark.node.HtmlInline; +import org.commonmark.node.Image; +import org.commonmark.node.IndentedCodeBlock; +import org.commonmark.node.Link; +import org.commonmark.node.ListItem; +import org.commonmark.node.Node; +import org.commonmark.node.OrderedList; +import org.commonmark.node.Paragraph; +import org.commonmark.node.SoftLineBreak; +import org.commonmark.node.StrongEmphasis; +import org.commonmark.node.Text; +import org.commonmark.node.ThematicBreak; +import org.commonmark.parser.Parser; + +/** + * Converts a markdown string into the {@code Document} proto's structured + * {@code Block}/{@code Inline} tree. + *

+ * Every Tika parser's SAX output already reaches this module as markdown text (the + * pipes default content handler renders {@code TikaCoreProperties.TIKA_CONTENT} as + * markdown via {@code ToMarkdownContentHandler}). Reusing that single rendering path + * plus commonmark-java's parser means the content tree is built once, generically, for + * every source format -- no per-format content handling is needed, only per-format + * metadata handling ({@link org.apache.tika.grpc.mapper.transform.DocumentTransformer}). + *

+ * Naming note: every {@code org.apache.tika.grpc.v1.*} proto type is referenced fully + * qualified in this file. Many proto message names (Heading, Paragraph, BlockQuote, + * Emphasis, Code, Link, Image, TableRow, TableCell, ...) collide with commonmark-java's + * own node class names, which are imported normally; fully qualifying one side avoids + * any ambiguity. + */ +public final class MarkdownBlockTreeBuilder { + + private static final List EXTENSIONS = + List.of(TablesExtension.create(), StrikethroughExtension.create()); + private static final Parser COMMONMARK = Parser.builder().extensions(EXTENSIONS).build(); + + private MarkdownBlockTreeBuilder() { + } + + /** Parses {@code markdown} into a flat list of top-level {@code Block}s. */ + public static List toBlocks(String markdown) { + if (markdown == null || markdown.isEmpty()) { + return List.of(); + } + return blockChildren(COMMONMARK.parse(markdown)); + } + + private static List blockChildren(Node parent) { + List blocks = new ArrayList<>(); + for (Node child = parent.getFirstChild(); child != null; child = child.getNext()) { + org.apache.tika.grpc.v1.Block block = toBlock(child); + if (block != null) { + blocks.add(block); + } + } + return blocks; + } + + private static org.apache.tika.grpc.v1.Block toBlock(Node node) { + if (node instanceof Heading) { + Heading heading = (Heading) node; + return org.apache.tika.grpc.v1.Block.newBuilder() + .setHeading(org.apache.tika.grpc.v1.Heading.newBuilder() + .setLevel(heading.getLevel()) + .addAllContent(inlineChildren(heading))) + .build(); + } + if (node instanceof Paragraph) { + return org.apache.tika.grpc.v1.Block.newBuilder() + .setParagraph(org.apache.tika.grpc.v1.Paragraph.newBuilder() + .addAllContent(inlineChildren(node))) + .build(); + } + if (node instanceof BlockQuote) { + return org.apache.tika.grpc.v1.Block.newBuilder() + .setBlockQuote(org.apache.tika.grpc.v1.BlockQuote.newBuilder() + .addAllBlocks(blockChildren(node))) + .build(); + } + if (node instanceof BulletList) { + BulletList bulletList = (BulletList) node; + return org.apache.tika.grpc.v1.Block.newBuilder() + .setBulletList(org.apache.tika.grpc.v1.BulletList.newBuilder() + .setMarker(String.valueOf(bulletList.getBulletMarker())) + .setTight(bulletList.isTight()) + .addAllItems(listItems(bulletList))) + .build(); + } + if (node instanceof OrderedList) { + OrderedList orderedList = (OrderedList) node; + return org.apache.tika.grpc.v1.Block.newBuilder() + .setOrderedList(org.apache.tika.grpc.v1.OrderedList.newBuilder() + .setStart(orderedList.getStartNumber()) + .setDelimiter(String.valueOf(orderedList.getDelimiter())) + .setTight(orderedList.isTight()) + .addAllItems(listItems(orderedList))) + .build(); + } + if (node instanceof FencedCodeBlock) { + FencedCodeBlock codeBlock = (FencedCodeBlock) node; + return org.apache.tika.grpc.v1.Block.newBuilder() + .setCodeBlock(org.apache.tika.grpc.v1.CodeBlock.newBuilder() + .setFenced(true) + .setInfo(nullToEmpty(codeBlock.getInfo())) + .setLiteral(nullToEmpty(codeBlock.getLiteral()))) + .build(); + } + if (node instanceof IndentedCodeBlock) { + return org.apache.tika.grpc.v1.Block.newBuilder() + .setCodeBlock(org.apache.tika.grpc.v1.CodeBlock.newBuilder() + .setFenced(false) + .setLiteral(nullToEmpty(((IndentedCodeBlock) node).getLiteral()))) + .build(); + } + if (node instanceof HtmlBlock) { + return org.apache.tika.grpc.v1.Block.newBuilder() + .setHtmlBlock(org.apache.tika.grpc.v1.HtmlBlock.newBuilder() + .setLiteral(nullToEmpty(((HtmlBlock) node).getLiteral()))) + .build(); + } + if (node instanceof ThematicBreak) { + return org.apache.tika.grpc.v1.Block.newBuilder() + .setThematicBreak(org.apache.tika.grpc.v1.ThematicBreak.getDefaultInstance()) + .build(); + } + if (node instanceof TableBlock) { + return org.apache.tika.grpc.v1.Block.newBuilder().setTable(toTable(node)).build(); + } + // Link reference definitions and any other unmodeled block produce no rendered output. + return null; + } + + private static List listItems(Node list) { + List items = new ArrayList<>(); + for (Node child = list.getFirstChild(); child != null; child = child.getNext()) { + if (child instanceof ListItem) { + items.add(org.apache.tika.grpc.v1.ListItem.newBuilder() + .addAllBlocks(blockChildren(child)) + .build()); + } + } + return items; + } + + private static org.apache.tika.grpc.v1.Table toTable(Node tableBlock) { + org.apache.tika.grpc.v1.Table.Builder table = org.apache.tika.grpc.v1.Table.newBuilder(); + for (Node section = tableBlock.getFirstChild(); section != null; section = section.getNext()) { + if (section instanceof TableHead) { + for (Node row = section.getFirstChild(); row != null; row = row.getNext()) { + if (!(row instanceof TableRow)) { + continue; + } + for (Node cell = row.getFirstChild(); cell != null; cell = cell.getNext()) { + if (cell instanceof TableCell) { + table.addHeader(toTableCell((TableCell) cell)); + } + } + } + } else if (section instanceof TableBody) { + for (Node row = section.getFirstChild(); row != null; row = row.getNext()) { + if (!(row instanceof TableRow)) { + continue; + } + org.apache.tika.grpc.v1.TableRow.Builder rowBuilder = org.apache.tika.grpc.v1.TableRow.newBuilder(); + for (Node cell = row.getFirstChild(); cell != null; cell = cell.getNext()) { + if (cell instanceof TableCell) { + rowBuilder.addCells(toTableCell((TableCell) cell)); + } + } + table.addRows(rowBuilder.build()); + } + } + } + return table.build(); + } + + private static org.apache.tika.grpc.v1.TableCell toTableCell(TableCell cell) { + return org.apache.tika.grpc.v1.TableCell.newBuilder() + .setAlignment(toAlignment(cell.getAlignment())) + .addAllContent(inlineChildren(cell)) + .build(); + } + + private static org.apache.tika.grpc.v1.Alignment toAlignment(TableCell.Alignment alignment) { + if (alignment == null) { + return org.apache.tika.grpc.v1.Alignment.ALIGN_NONE; + } + switch (alignment) { + case LEFT: + return org.apache.tika.grpc.v1.Alignment.ALIGN_LEFT; + case CENTER: + return org.apache.tika.grpc.v1.Alignment.ALIGN_CENTER; + case RIGHT: + return org.apache.tika.grpc.v1.Alignment.ALIGN_RIGHT; + default: + return org.apache.tika.grpc.v1.Alignment.ALIGN_NONE; + } + } + + private static List inlineChildren(Node parent) { + List inlines = new ArrayList<>(); + for (Node child = parent.getFirstChild(); child != null; child = child.getNext()) { + org.apache.tika.grpc.v1.Inline inline = toInline(child); + if (inline != null) { + inlines.add(inline); + } + } + return inlines; + } + + private static org.apache.tika.grpc.v1.Inline toInline(Node node) { + if (node instanceof Text) { + return org.apache.tika.grpc.v1.Inline.newBuilder().setText(((Text) node).getLiteral()).build(); + } + if (node instanceof Emphasis) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setEmphasis(org.apache.tika.grpc.v1.Emphasis.newBuilder().addAllContent(inlineChildren(node))) + .build(); + } + if (node instanceof StrongEmphasis) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setStrong(org.apache.tika.grpc.v1.Strong.newBuilder().addAllContent(inlineChildren(node))) + .build(); + } + if (node instanceof Strikethrough) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setStrikethrough(org.apache.tika.grpc.v1.Strikethrough.newBuilder().addAllContent(inlineChildren(node))) + .build(); + } + if (node instanceof Code) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setCode(org.apache.tika.grpc.v1.Code.newBuilder().setLiteral(((Code) node).getLiteral())) + .build(); + } + if (node instanceof Link) { + Link link = (Link) node; + org.apache.tika.grpc.v1.Link.Builder linkBuilder = org.apache.tika.grpc.v1.Link.newBuilder() + .setDestination(nullToEmpty(link.getDestination())) + .addAllContent(inlineChildren(link)); + if (link.getTitle() != null) { + linkBuilder.setTitle(link.getTitle()); + } + return org.apache.tika.grpc.v1.Inline.newBuilder().setLink(linkBuilder).build(); + } + if (node instanceof Image) { + Image image = (Image) node; + org.apache.tika.grpc.v1.Image.Builder imageBuilder = org.apache.tika.grpc.v1.Image.newBuilder() + .setDestination(nullToEmpty(image.getDestination())) + .setAlt(collectText(image)); + if (image.getTitle() != null) { + imageBuilder.setTitle(image.getTitle()); + } + return org.apache.tika.grpc.v1.Inline.newBuilder().setImage(imageBuilder).build(); + } + if (node instanceof HardLineBreak) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setLineBreak(org.apache.tika.grpc.v1.LineBreak.newBuilder().setHard(true)) + .build(); + } + if (node instanceof SoftLineBreak) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setLineBreak(org.apache.tika.grpc.v1.LineBreak.newBuilder().setHard(false)) + .build(); + } + if (node instanceof HtmlInline) { + return org.apache.tika.grpc.v1.Inline.newBuilder().setHtml(((HtmlInline) node).getLiteral()).build(); + } + // Reference definitions and any other unmodeled inline node produce no rendered output. + return null; + } + + /** Flattens inline content to plain text (for image alt), losing no literals. */ + private static String collectText(Node node) { + StringBuilder sb = new StringBuilder(); + for (Node child = node.getFirstChild(); child != null; child = child.getNext()) { + if (child instanceof Text) { + sb.append(((Text) child).getLiteral()); + } else if (child instanceof Code) { + sb.append(((Code) child).getLiteral()); + } else if (child instanceof HtmlInline) { + sb.append(((HtmlInline) child).getLiteral()); + } else if (child instanceof SoftLineBreak || child instanceof HardLineBreak) { + sb.append(' '); + } else { + sb.append(collectText(child)); + } + } + return sb.toString(); + } + + private static String nullToEmpty(String value) { + return value == null ? "" : value; + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformer.java new file mode 100644 index 0000000000..ef5f913339 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformer.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.metadata.XMPRights; + +/** + * Creative Commons / XMP rights detection. Unlike the format transformers, this one is + * cross-cutting: Creative Commons or other rights-management metadata can ride along with + * any document format (a PDF, a JPEG, an Office doc, ...), so it does not key off + * content-type at all. Instead it inspects the full metadata for CC/license-shaped field + * names and values, or for any populated XMP rights-management property. + * + *

Rights-management specifics (marked, owner, usage terms, certificate, ...) are NOT + * given their own proto fields. They flow into the tagged tail, typed where Tika declares + * the type and string otherwise - same philosophy as the format transformers. + */ +public final class CreativeCommonsDocumentTransformer implements DocumentTransformer { + + private static final String[] NAME_HINTS = {"license", "creative", "cc:", "rights"}; + + private static final Property[] XMP_RIGHTS_PROPERTIES = { + XMPRights.MARKED, + XMPRights.OWNER, + XMPRights.USAGE_TERMS, + XMPRights.WEB_STATEMENT, + XMPRights.CERTIFICATE + }; + + @Override + public boolean isCrossCutting() { + return true; + } + + @Override + public boolean appliesTo(Metadata tika) { + for (String name : tika.names()) { + String lowerName = name.toLowerCase(Locale.ROOT); + for (String hint : NAME_HINTS) { + if (lowerName.contains(hint)) { + String value = tika.get(name); + if (value != null && value.toLowerCase(Locale.ROOT).contains("creative")) { + return true; + } + break; + } + } + } + + for (Property property : XMP_RIGHTS_PROPERTIES) { + String value = tika.get(property); + if (value != null && !value.trim().isEmpty()) { + return true; + } + } + + return false; + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + TransformSupport.setString(tika, TikaCoreProperties.RIGHTS, meta::setRights, consumed); + // A CC license URL, e.g. https://creativecommons.org/licenses/by/4.0/, typically + // lives in the XMP rights "web statement" property. + TransformSupport.setString(tika, XMPRights.WEB_STATEMENT, meta::setLicenseUrl, consumed); + + // Everything else (xmpRights:Marked, xmpRights:Owner, xmpRights:UsageTerms, + // xmpRights:Certificate, ...) lands in the tagged tail, appended once by + // DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformer.java new file mode 100644 index 0000000000..80bacf815c --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformer.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; + +/** + * Maps one Tika {@link Metadata} (a document, or an embedded part) onto the typed + * {@link Document}. Transformers are code, not schema: adding a parser adds a + * transformer, and the wire contract never changes. More than one transformer may + * apply to a single document (a format transformer plus cross-cutting ones, such as + * Creative Commons rights detection alongside a PDF or Office transformer). + */ +public interface DocumentTransformer { + + /** + * True if this transformer applies to the given document. Most transformers only + * look at {@code tika.get(Metadata.CONTENT_TYPE)}; cross-cutting ones (e.g. Creative + * Commons rights detection) may inspect the full metadata instead. + */ + boolean appliesTo(Metadata tika); + + /** + * Populate typed fields on the builder, and mark every Tika key consumed in + * {@code consumed}. {@code consumed} is shared across every transformer that applies to + * this document (not private to this transformer), so the tagged tail -- appended once, + * after every applicable transformer has run -- never duplicates a key that a different + * transformer already mapped to a typed field. + */ + void transform(Metadata tika, Document.Builder document, Set consumed); + + /** + * True if this transformer is cross-cutting (may apply alongside, not instead of, a + * format transformer -- e.g. Creative Commons rights detection). Cross-cutting + * transformers do not count toward whether a document "matched" a format, so they never + * suppress the generic fallback: a plain-text document with CC rights metadata still + * gets the universal title/author/description mapping. + */ + default boolean isCrossCutting() { + return false; + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformers.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformers.java new file mode 100644 index 0000000000..d97e740c0b --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformers.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; + +/** + * Picks and runs the applicable transformer(s) for a parsed document. Multiple may + * apply (e.g. a format transformer plus the Creative Commons rights transformer); the + * generic fallback runs whenever no format-specific transformer matched, so an unknown + * format still yields a useful, lossless Document. + * + * Adding support for a new format means adding a transformer to {@link #defaults()}. + * The proto does not change, so clients never rebuild for it. + */ +public final class DocumentTransformers { + + private final List transformers; + private final DocumentTransformer fallback = new GenericDocumentTransformer(); + + public DocumentTransformers(List transformers) { + this.transformers = transformers; + } + + public static DocumentTransformers defaults() { + return new DocumentTransformers(List.of( + new PdfDocumentTransformer(), + new OfficeDocumentTransformer(), + new ImageDocumentTransformer(), + new HtmlDocumentTransformer(), + new RtfDocumentTransformer(), + new EpubDocumentTransformer(), + new WarcDocumentTransformer(), + new CreativeCommonsDocumentTransformer() + )); + } + + /** Populate the typed metadata/tagged-tail portion of {@code document} for {@code tika}. */ + public void transform(Metadata tika, Document.Builder document) { + // Shared across every applicable transformer so the tagged tail, appended exactly + // once below, never duplicates a key that a different transformer already typed. + Set consumed = new HashSet<>(); + boolean matchedFormat = false; + for (DocumentTransformer transformer : transformers) { + if (transformer.appliesTo(tika)) { + transformer.transform(tika, document, consumed); + matchedFormat |= !transformer.isCrossCutting(); + } + } + if (!matchedFormat) { + fallback.transform(tika, document, consumed); + } + MetadataTagger.appendTail(tika, consumed, document); + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java new file mode 100644 index 0000000000..8300d8df80 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * EPUB. Maps the common, cross-format facts into typed DocumentMetadata. + * + * EPUB-specific properties (rendition layout, spec version, OPF/NAV structural fields, ...) + * are NOT given their own proto fields. They flow into the tagged tail, typed where Tika + * declares the type and string otherwise. That is the whole point: format richness lives + * in code plus the tail, not in the wire contract - so the schema stays small and stable + * and clients never rebuild when we add or change an EPUB property. + */ +public final class EpubDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + return contentType != null && contentType.toLowerCase(Locale.ROOT).contains("epub"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); + TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.SUBJECT, meta::addAllKeywords, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + + // Everything EPUB-specific (epub:*, dc:identifier, OPF/NAV structural fields, ...) + // lands in the tagged tail, appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/FormatCategoryDetector.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/FormatCategoryDetector.java new file mode 100644 index 0000000000..0e570edde8 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/FormatCategoryDetector.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; + +import org.apache.tika.grpc.v1.FormatCategory; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * Detects the coarse {@link FormatCategory} routing hint from Tika metadata. This is + * independent of which {@link DocumentTransformer}s actually run -- transformers + * self-select via {@link DocumentTransformer#appliesTo(String)} and are not mutually + * exclusive (e.g. Creative Commons rights can coexist with any category here), so this + * enum exists purely as a cheap client-side routing hint. + */ +public final class FormatCategoryDetector { + + private FormatCategoryDetector() { + } + + public static FormatCategory detect(Metadata metadata) { + String mimeType = metadata.get(Metadata.CONTENT_TYPE); + String resourceName = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY); + + if (mimeType != null) { + String mime = mimeType.toLowerCase(Locale.ROOT); + if (mime.contains("pdf")) { + return FormatCategory.FORMAT_CATEGORY_PDF; + } + if (mime.contains("officedocument") || mime.contains("msword") + || mime.contains("ms-excel") || mime.contains("ms-powerpoint") + || mime.contains("opendocument") || mime.contains("vnd.ms-") + || mime.contains("vnd.openxmlformats")) { + return FormatCategory.FORMAT_CATEGORY_OFFICE; + } + if (mime.startsWith("image/")) { + return FormatCategory.FORMAT_CATEGORY_IMAGE; + } + if (mime.contains("text/html") || mime.contains("application/xhtml")) { + return FormatCategory.FORMAT_CATEGORY_HTML; + } + if (mime.contains("rtf")) { + return FormatCategory.FORMAT_CATEGORY_RTF; + } + if (mime.contains("epub")) { + return FormatCategory.FORMAT_CATEGORY_EPUB; + } + if (mime.contains("warc")) { + return FormatCategory.FORMAT_CATEGORY_WARC; + } + } + + if (resourceName != null) { + String name = resourceName.toLowerCase(Locale.ROOT); + if (name.endsWith(".pdf")) { + return FormatCategory.FORMAT_CATEGORY_PDF; + } + if (name.endsWith(".doc") || name.endsWith(".docx") || name.endsWith(".xls") + || name.endsWith(".xlsx") || name.endsWith(".ppt") || name.endsWith(".pptx") + || name.endsWith(".odt") || name.endsWith(".ods") || name.endsWith(".odp")) { + return FormatCategory.FORMAT_CATEGORY_OFFICE; + } + if (name.endsWith(".jpg") || name.endsWith(".jpeg") || name.endsWith(".png") + || name.endsWith(".gif") || name.endsWith(".tiff") || name.endsWith(".tif") + || name.endsWith(".bmp")) { + return FormatCategory.FORMAT_CATEGORY_IMAGE; + } + if (name.endsWith(".html") || name.endsWith(".htm") || name.endsWith(".xhtml")) { + return FormatCategory.FORMAT_CATEGORY_HTML; + } + if (name.endsWith(".rtf")) { + return FormatCategory.FORMAT_CATEGORY_RTF; + } + if (name.endsWith(".epub")) { + return FormatCategory.FORMAT_CATEGORY_EPUB; + } + if (name.endsWith(".warc") || name.endsWith(".arc")) { + return FormatCategory.FORMAT_CATEGORY_WARC; + } + } + + return FormatCategory.FORMAT_CATEGORY_GENERIC; + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java new file mode 100644 index 0000000000..beb833df35 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * Always-applicable fallback. Pulls the universal Dublin Core fields into typed metadata + * and routes everything else to the tagged tail, so an unknown or unsupported format + * still produces a useful, lossless Document. + */ +public final class GenericDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + return true; + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); + TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.SUBJECT, meta::addAllKeywords, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java new file mode 100644 index 0000000000..a9ff5cb966 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * HTML / XHTML. Maps the common, cross-format facts into typed DocumentMetadata. + * + * HTML-specific properties (html:meta:*, Open Graph og:*, Twitter Card twitter:*, + * html:link:*, ICBM geo, encoding-detector fields, ...) are NOT given their own proto + * fields. They flow into the tagged tail, typed where Tika declares the type and string + * otherwise. That is the whole point: format richness lives in code plus the tail, not + * in the wire contract - so the schema stays small and stable and clients never rebuild + * when we add or change an HTML property. + */ +public final class HtmlDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + if (contentType == null) { + return false; + } + String lower = contentType.toLowerCase(Locale.ROOT); + return lower.contains("text/html") || lower.contains("application/xhtml"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); + TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.SUBJECT, meta::addAllKeywords, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + + // Everything HTML-specific (html:meta:*, og:*, twitter:*, html:link:*, ICBM, ...) + // lands in the tagged tail, appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java new file mode 100644 index 0000000000..cf9bcd6e15 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.IPTC; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TIFF; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * Images. Maps the common, cross-format facts into typed DocumentMetadata. + * + * Image-specific properties (EXIF exposure/f-number/focal-length/flash/ISO, GPS + * coordinates, IPTC headline/category/credit-line/copyright-notice, Photoshop + * city/country/state, equipment make/model, resolution, orientation, ...) are NOT + * given their own proto fields. They flow into the tagged tail, typed where Tika + * declares the type and string otherwise. That is the whole point: format richness + * lives in code plus the tail, not in the wire contract - so the schema stays small + * and stable and clients never rebuild when we add or change an image property. + */ +public final class ImageDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + return contentType != null && contentType.toLowerCase(Locale.ROOT).startsWith("image/"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); + TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); + TransformSupport.addStrings(tika, IPTC.KEYWORDS, meta::addAllKeywords, consumed); + TransformSupport.setTimestamp(tika, TIFF.ORIGINAL_DATE, meta::setCreated, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + TransformSupport.setInt(tika, TIFF.IMAGE_WIDTH, meta::setWidth, consumed); + TransformSupport.setInt(tika, TIFF.IMAGE_LENGTH, meta::setHeight, consumed); + + // Everything image-specific (exif:*, tiff:*, IPTC-*, photoshop:*, geo:*, ...) lands + // in the tagged tail, appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java new file mode 100644 index 0000000000..855c384d65 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.time.Instant; +import java.util.Date; +import java.util.Locale; +import java.util.Set; + +import com.google.protobuf.Timestamp; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.MetadataField; +import org.apache.tika.grpc.v1.MetadataValue; +import org.apache.tika.grpc.v1.StringList; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; + +/** + * Builds the tagged metadata tail. The tag comes from Tika's declared + * {@link Property} value type. Keys with no declared type are emitted as strings, + * never guessed - so we never turn "8 8 8 8" into a broken integer. + */ +public final class MetadataTagger { + + private MetadataTagger() { + } + + /** Append every key not already consumed by a typed field into {@code Document.extra}. */ + public static void appendTail(Metadata tika, Set consumed, Document.Builder document) { + for (String key : tika.names()) { + if (consumed.contains(key)) { + continue; + } + MetadataValue value = tag(tika, key); + if (value != null) { + document.addExtra(MetadataField.newBuilder().setKey(key).setValue(value).build()); + } + } + } + + /** Tag one key by its declared Property type; fall back to multivalue strings. */ + public static MetadataValue tag(Metadata tika, String key) { + String[] values = tika.getValues(key); + if (values == null || values.length == 0) { + return null; + } + Property property = Property.get(key); + if (values.length == 1 && property != null) { + MetadataValue typed = tagByDeclaredType(tika, property, values[0].trim()); + if (typed != null) { + return typed; + } + } + StringList.Builder strings = StringList.newBuilder(); + for (String v : values) { + if (v != null && !v.trim().isEmpty()) { + strings.addValues(v.trim()); + } + } + return strings.getValuesCount() == 0 + ? null + : MetadataValue.newBuilder().setStrings(strings).build(); + } + + private static MetadataValue tagByDeclaredType(Metadata tika, Property property, String raw) { + switch (property.getPrimaryProperty().getValueType()) { + case INTEGER: + try { + return MetadataValue.newBuilder().setInteger(Long.parseLong(raw)).build(); + } catch (NumberFormatException ignored) { + return null; + } + case REAL: + case RATIONAL: + try { + return MetadataValue.newBuilder().setNumber(Double.parseDouble(raw)).build(); + } catch (NumberFormatException ignored) { + return null; + } + case BOOLEAN: + return MetadataValue.newBuilder().setBoolean(parseBoolean(raw)).build(); + case DATE: + Date date = tika.getDate(property); + return date == null + ? null + : MetadataValue.newBuilder().setTimestamp(toTimestamp(date)).build(); + default: + return null; // not a declared scalar -> caller falls back to strings + } + } + + static boolean parseBoolean(String v) { + String s = v.trim().toLowerCase(Locale.ROOT); + return "true".equals(s) || "yes".equals(s) || "1".equals(s); + } + + static Timestamp toTimestamp(Date date) { + Instant i = date.toInstant(); + return Timestamp.newBuilder().setSeconds(i.getEpochSecond()).setNanos(i.getNano()).build(); + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java new file mode 100644 index 0000000000..bbc899ba13 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Office; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * Office (MS Word/Excel/PowerPoint, OOXML, OpenDocument). Maps the common, cross-format + * facts into typed DocumentMetadata. + * + * Office-specific properties (slide/paragraph/line/table/image/object counts, OOXML + * core/extended properties, hidden sheets/slides, comments, track changes, ...) are NOT + * given their own proto fields. They flow into the tagged tail, typed where Tika declares + * the type and string otherwise. That is the whole point: format richness lives in code + * plus the tail, not in the wire contract - so the schema stays small and stable and + * clients never rebuild when we add or change an Office property. + */ +public final class OfficeDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + if (contentType == null) { + return false; + } + String lower = contentType.toLowerCase(Locale.ROOT); + return lower.contains("officedocument") + || lower.contains("msword") + || lower.contains("ms-excel") + || lower.contains("ms-powerpoint") + || lower.contains("opendocument") + || lower.contains("vnd.ms-") + || lower.contains("vnd.openxmlformats"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); + TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); + TransformSupport.addStrings(tika, Office.KEYWORDS, meta::addAllKeywords, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + TransformSupport.setInt(tika, Office.PAGE_COUNT, meta::setPageCount, consumed); + TransformSupport.setLong(tika, Office.WORD_COUNT, meta::setWordCount, consumed); + TransformSupport.setLong(tika, Office.CHARACTER_COUNT, meta::setCharacterCount, consumed); + + // Everything Office-specific (slide/paragraph/line/table/image/object counts, + // OOXML core/extended properties, hidden sheets/slides, comments, track changes, ...) + // lands in the tagged tail, appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java new file mode 100644 index 0000000000..a39131d2b2 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Office; +import org.apache.tika.metadata.PagedText; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * PDF. Maps the common, cross-format facts into typed DocumentMetadata. + * + * PDF-specific properties (encryption flags, XMP, permissions, incremental updates, ...) + * are NOT given their own proto fields. They flow into the tagged tail, typed where Tika + * declares the type and string otherwise. That is the whole point: format richness lives + * in code plus the tail, not in the wire contract - so the schema stays small and stable + * and clients never rebuild when we add or change a PDF property. + */ +public final class PdfDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + return contentType != null && contentType.toLowerCase(Locale.ROOT).contains("pdf"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); + TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.SUBJECT, meta::addAllKeywords, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + TransformSupport.setInt(tika, PagedText.N_PAGES, meta::setPageCount, consumed); + TransformSupport.setLong(tika, Office.WORD_COUNT, meta::setWordCount, consumed); + + // Everything PDF-specific (pdf:*, xmpPDF:*, access_permission:*, ...) lands in the + // tagged tail, appended once by DocumentTransformers after every applicable + // transformer has contributed to `consumed`. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java new file mode 100644 index 0000000000..6090e8f255 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Office; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * RTF. Maps the common, cross-format facts into typed DocumentMetadata. + * + * RTF-specific properties (encapsulated HTML flags, embedded-object thumbnails and + * application/class/topic/item strings, OOXML core/extended fields such as category, + * manager, company, template, ...) are NOT given their own proto fields. They flow into + * the tagged tail, typed where Tika declares the type and string otherwise. That is the + * whole point: format richness lives in code plus the tail, not in the wire contract - so + * the schema stays small and stable and clients never rebuild when we add or change an + * RTF property. + */ +public final class RtfDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + return contentType != null && contentType.toLowerCase(Locale.ROOT).contains("rtf"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); + TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); + TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); + TransformSupport.addStrings(tika, Office.KEYWORDS, meta::addAllKeywords, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + TransformSupport.setInt(tika, Office.PAGE_COUNT, meta::setPageCount, consumed); + TransformSupport.setLong(tika, Office.WORD_COUNT, meta::setWordCount, consumed); + TransformSupport.setLong(tika, Office.CHARACTER_COUNT, meta::setCharacterCount, consumed); + + // Everything RTF-specific (rtf_meta:*, extended-properties:*, ...) lands in the + // tagged tail, appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java new file mode 100644 index 0000000000..541bb7e147 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import com.google.protobuf.Timestamp; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; + +/** + * Helpers for transformers: set a typed Document field from a Tika Property and mark the + * key consumed so it is not duplicated into the tagged tail. + */ +public final class TransformSupport { + + private TransformSupport() { + } + + public static void setString(Metadata md, Property key, Consumer setter, Set consumed) { + String v = md.get(key); + if (v != null && !v.trim().isEmpty()) { + setter.accept(v.trim()); + consumed.add(key.getName()); + } + } + + public static void addStrings(Metadata md, Property key, Consumer> setter, Set consumed) { + String[] values = md.getValues(key); + if (values != null && values.length > 0) { + List list = Arrays.stream(values) + .filter(s -> s != null && !s.trim().isEmpty()) + .map(String::trim) + .collect(Collectors.toList()); + if (!list.isEmpty()) { + setter.accept(list); + consumed.add(key.getName()); + } + } + } + + public static void setInt(Metadata md, Property key, Consumer setter, Set consumed) { + Integer v = md.getInt(key); + if (v != null) { + setter.accept(v); + consumed.add(key.getName()); + } + } + + public static void setLong(Metadata md, Property key, Consumer setter, Set consumed) { + Integer v = md.getInt(key); + if (v != null) { + setter.accept(v.longValue()); + consumed.add(key.getName()); + } + } + + public static void setDouble(Metadata md, Property key, Consumer setter, Set consumed) { + String v = md.get(key); + if (v != null && !v.trim().isEmpty()) { + try { + setter.accept(Double.parseDouble(v.trim())); + consumed.add(key.getName()); + } catch (NumberFormatException ignored) { + // leave unconsumed; falls through to the tagged tail + } + } + } + + public static void setTimestamp(Metadata md, Property key, Consumer setter, Set consumed) { + Date d = md.getDate(key); + if (d != null) { + setter.accept(MetadataTagger.toTimestamp(d)); + consumed.add(key.getName()); + } + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformer.java new file mode 100644 index 0000000000..195e39f779 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformer.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; + +/** + * WARC (Web ARChive) containers. Unlike most formats, WARC's useful facts - WARC-Date, + * WARC-Target-URI, WARC-Record-ID, HTTP status, HTTP response headers, ... - are not + * normalized by {@code org.apache.tika.parser.warc.WARCParser} onto declared + * {@link org.apache.tika.metadata.Property} constants. The parser adds them under literal + * string keys instead ({@code warc:WARC-Date}, {@code warc:WARC-Target-URI}, + * {@code warc:http:status}, {@code warc:http:*}, ...). {@code org.apache.tika.metadata.WARC} + * only declares a handful of properties (WARC_RECORD_CONTENT_TYPE, WARC_PAYLOAD_CONTENT_TYPE, + * WARC_RECORD_ID, WARC_WARNING), and none of the common, cross-format facts that this + * transformer could map (title, created, ...) are actually populated by the parser for a + * WARC record. + * + * There is therefore nothing here that fairly maps onto the small set of typed + * DocumentMetadata fields - forcing one would be dishonest, not useful. That is fine: the + * tagged tail already handles arbitrary string keys, typed by Tika's declared Property type + * where one exists and as plain strings otherwise, so none of the WARC/HTTP richness is + * lost - it just does not need its own proto fields. + */ +public final class WarcDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + return contentType != null && contentType.toLowerCase(Locale.ROOT).contains("warc"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set consumed) { + // No common field fairly maps to the typed DocumentMetadata for a WARC record (see + // class javadoc). Everything (warc:WARC-Date, warc:WARC-Target-URI, + // warc:WARC-Record-ID, warc:http:status, warc:http:*, ...) lands in the tagged tail, + // appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ClasspathTestDocuments.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ClasspathTestDocuments.java new file mode 100644 index 0000000000..fba712e0a1 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ClasspathTestDocuments.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper; + +import java.io.IOException; +import java.net.JarURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +/** + * Discovers {@code test-documents/*} fixtures contributed by parser module test-jars on the classpath. + */ +final class ClasspathTestDocuments { + + private static final String TEST_DOCUMENTS_DIR = "test-documents"; + + private ClasspathTestDocuments() { + } + + static List listByExtension(String extension) throws IOException { + String suffix = extension.startsWith(".") ? extension : "." + extension; + Set found = new TreeSet<>(); + ClassLoader classLoader = ClasspathTestDocuments.class.getClassLoader(); + Enumeration roots = classLoader.getResources(TEST_DOCUMENTS_DIR); + while (roots.hasMoreElements()) { + collectFromUrl(roots.nextElement(), suffix, found); + } + return new ArrayList<>(found); + } + + static List listByExtensions(String... extensions) throws IOException { + Set found = new TreeSet<>(); + for (String extension : extensions) { + found.addAll(listByExtension(extension)); + } + return new ArrayList<>(found); + } + + private static void collectFromUrl(URL url, String suffix, Set found) throws IOException { + if ("file".equals(url.getProtocol())) { + try { + Path dir = Paths.get(url.toURI()); + if (Files.isDirectory(dir)) { + collectFromDirectory(dir, suffix, found); + } + } catch (java.net.URISyntaxException e) { + throw new IOException("Bad test-documents URL: " + url, e); + } + return; + } + if ("jar".equals(url.getProtocol())) { + collectFromJarUrl(url, suffix, found); + } + } + + private static void collectFromDirectory(Path dir, String suffix, Set found) throws IOException { + try (var paths = Files.walk(dir)) { + paths.filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().endsWith(suffix)) + .forEach(path -> found.add(path.getFileName().toString())); + } + } + + private static void collectFromJarUrl(URL url, String suffix, Set found) throws IOException { + JarURLConnection connection = (JarURLConnection) url.openConnection(); + String prefix = connection.getEntryName(); + if (prefix == null) { + prefix = TEST_DOCUMENTS_DIR; + } + if (!prefix.endsWith("/")) { + prefix = prefix + "/"; + } + JarFile jarFile = connection.getJarFile(); + final String entryPrefix = prefix; + for (JarEntry entry : jarFile.stream().toList()) { + if (entry.isDirectory()) { + continue; + } + String name = entry.getName(); + if (!name.startsWith(entryPrefix) || !name.endsWith(suffix)) { + continue; + } + int slash = name.lastIndexOf('/'); + found.add(slash >= 0 ? name.substring(slash + 1) : name); + } + } + +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java new file mode 100644 index 0000000000..59c4411e23 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.FormatCategory; +import org.apache.tika.grpc.v1.ParseStatus; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * End-to-end coverage of {@link DocumentBuilder}: the content tree (parsed from the + * pipes-default markdown rendering), typed/tagged metadata (via {@link + * org.apache.tika.grpc.mapper.transform.DocumentTransformers}), format-category routing, + * status, and embedded-document recursion. + */ +class DocumentBuilderTest extends ParseFixtureSupport { + + @Test + void buildsContentTreeAndMetadataFromRealPdf() throws Exception { + Document document = map(parseBody("testPDF.pdf"), "doc-1", 42L); + + assertTrue(document.getMarkdown().contains("Tika")); + assertFalse(document.getBlocksList().isEmpty()); + assertEquals(FormatCategory.FORMAT_CATEGORY_PDF, document.getFormatCategory()); + assertEquals("application/pdf", document.getContentType()); + assertEquals("Apache Tika - Apache Tika", document.getMetadata().getTitle()); + assertTrue(document.getExtraCount() > 0); + + assertEquals(ParseStatus.Status.SUCCESS, document.getStatus().getStatus()); + assertEquals(42L, document.getStatus().getParseTimeMs()); + } + + @Test + void recursesIntoEmbeddedDocuments() { + Metadata primary = new Metadata(); + primary.set(Metadata.CONTENT_TYPE, "application/pdf"); + primary.set(TikaCoreProperties.TITLE, "Container"); + + Metadata embedded = new Metadata(); + embedded.set(Metadata.CONTENT_TYPE, "image/jpeg"); + embedded.set(TikaCoreProperties.RESOURCE_NAME_KEY, "photo.jpg"); + embedded.set(TikaCoreProperties.TIKA_CONTENT, "an embedded photo"); + + Document document = DocumentBuilder.build( + primary, List.of(primary, embedded), "# Container\n", "doc-1", "PARSE_SUCCESS", 5L); + + assertEquals(1, document.getEmbeddedCount()); + Document child = document.getEmbedded(0); + assertEquals("image/jpeg", child.getContentType()); + assertEquals(FormatCategory.FORMAT_CATEGORY_IMAGE, child.getFormatCategory()); + assertEquals("photo.jpg", child.getOrigin().getFilename()); + assertEquals("an embedded photo", child.getMarkdown()); + } + + @Test + void reportsFailureWhenNoMetadata() { + Document document = DocumentBuilder.build(null, null, null, "doc-1", "PARSE_EXCEPTION", 1L); + + assertEquals(ParseStatus.Status.FAILED, document.getStatus().getStatus()); + assertFalse(document.getStatus().getErrorsList().isEmpty()); + } + + /** + * {@code pipesStatus} is populated from {@code PipesResult.RESULT_STATUS.name()} (see + * org.apache.tika.pipes.api.PipesResult), never from a plain "OK" -- there is no such + * enum constant. A real successful parse reports as e.g. "PARSE_SUCCESS" or + * "EMIT_SUCCESS", and a real timeout reports as "TIMEOUT" (categorized as a process + * crash, not a partial success). The status mapping must recognize the real names. + */ + @Test + void mapsRealPipesResultStatusNames() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "text/plain"); + + assertEquals(ParseStatus.Status.SUCCESS, + DocumentBuilder.build(metadata, null, null, "d", "PARSE_SUCCESS", 1L).getStatus().getStatus(), + "PARSE_SUCCESS is a real, clean success status"); + assertEquals(ParseStatus.Status.SUCCESS, + DocumentBuilder.build(metadata, null, null, "d", "EMIT_SUCCESS", 1L).getStatus().getStatus()); + assertEquals(ParseStatus.Status.SUCCESS, + DocumentBuilder.build(metadata, null, null, "d", "EMPTY_OUTPUT", 1L).getStatus().getStatus()); + + assertEquals(ParseStatus.Status.PARTIAL, + DocumentBuilder.build(metadata, null, null, "d", "PARSE_SUCCESS_WITH_EXCEPTION", 1L) + .getStatus().getStatus(), + "succeeded, but with a caveat along the way, is a partial success not a clean one"); + + assertEquals(ParseStatus.Status.FAILED, + DocumentBuilder.build(metadata, null, null, "d", "TIMEOUT", 1L).getStatus().getStatus(), + "TIMEOUT is categorized as a process crash, not a partial success"); + assertEquals(ParseStatus.Status.FAILED, + DocumentBuilder.build(metadata, null, null, "d", "OOM", 1L).getStatus().getStatus()); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java new file mode 100644 index 0000000000..cb2bad1a79 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper; + +import java.util.List; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.ToMarkdownContentHandler; + +/** + * Shared helpers: parse Tika test fixtures and map them through {@link DocumentBuilder}. + */ +public abstract class ParseFixtureSupport extends TikaTest { + + /** + * Result of parsing a fixture with markdown body extraction enabled. + */ + public record ParseFixture(Metadata primary, List allMetadata, String body) { + } + + protected ParseFixture parseBody(String fileName) throws Exception { + java.io.StringWriter writer = new java.io.StringWriter(); + ToMarkdownContentHandler handler = new ToMarkdownContentHandler(writer); + Metadata metadata = new Metadata(); + try (TikaInputStream input = getResourceAsStream("/test-documents/" + fileName)) { + AUTO_DETECT_PARSER.parse(input, handler, metadata, new ParseContext()); + } + return new ParseFixture(metadata, List.of(metadata), writer.toString()); + } + + protected Document map(ParseFixture fixture, String docId, long parseTimeMs) { + // "PARSE_SUCCESS" mirrors a real org.apache.tika.pipes.api.PipesResult.RESULT_STATUS + // name -- there is no "OK" constant on that enum. + return DocumentBuilder.build( + fixture.primary(), fixture.allMetadata(), fixture.body(), docId, "PARSE_SUCCESS", parseTimeMs); + } + + protected Document map(ParseFixture fixture, String docId) { + return map(fixture, docId, 1L); + } + +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilderTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilderTest.java new file mode 100644 index 0000000000..39c9c2ee4e --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilderTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.content; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.grpc.v1.Alignment; +import org.apache.tika.grpc.v1.Block; +import org.apache.tika.grpc.v1.Inline; + +class MarkdownBlockTreeBuilderTest { + + @Test + void emptyOrNullMarkdownYieldsNoBlocks() { + assertTrue(MarkdownBlockTreeBuilder.toBlocks(null).isEmpty()); + assertTrue(MarkdownBlockTreeBuilder.toBlocks("").isEmpty()); + } + + @Test + void nestedBulletListInsideAnOrderedListItemIsPreserved() { + String markdown = "1. first\n" + + " - nested a\n" + + " - nested b\n" + + "2. second\n"; + + List blocks = MarkdownBlockTreeBuilder.toBlocks(markdown); + assertEquals(1, blocks.size()); + Block ordered = blocks.get(0); + assertTrue(ordered.hasOrderedList()); + assertEquals(2, ordered.getOrderedList().getItemsCount()); + + org.apache.tika.grpc.v1.ListItem first = ordered.getOrderedList().getItems(0); + boolean sawNestedBulletList = first.getBlocksList().stream().anyMatch(Block::hasBulletList); + assertTrue(sawNestedBulletList, "the nested bullet list under item 1 must survive the recursive conversion"); + + org.apache.tika.grpc.v1.BulletList nested = first.getBlocksList().stream() + .filter(Block::hasBulletList) + .map(Block::getBulletList) + .findFirst() + .orElseThrow(); + assertEquals(2, nested.getItemsCount()); + } + + @Test + void gfmTableWithAlignmentAndMultipleRows() { + String markdown = "| Left | Center | Right |\n" + + "|:---|:---:|---:|\n" + + "| a | b | c |\n" + + "| d | e | f |\n"; + + List blocks = MarkdownBlockTreeBuilder.toBlocks(markdown); + assertEquals(1, blocks.size()); + assertTrue(blocks.get(0).hasTable()); + + org.apache.tika.grpc.v1.Table table = blocks.get(0).getTable(); + assertEquals(3, table.getHeaderCount()); + assertEquals(Alignment.ALIGN_LEFT, table.getHeader(0).getAlignment()); + assertEquals(Alignment.ALIGN_CENTER, table.getHeader(1).getAlignment()); + assertEquals(Alignment.ALIGN_RIGHT, table.getHeader(2).getAlignment()); + + assertEquals(2, table.getRowsCount()); + assertEquals(3, table.getRows(0).getCellsCount()); + assertEquals(3, table.getRows(1).getCellsCount()); + } + + @Test + void linkAndImageCarryDestinationAndOptionalTitle() { + String markdown = "[link text](https://example.com/page \"a title\")\n\n" + + "![alt text](https://example.com/img.png)\n"; + + List blocks = MarkdownBlockTreeBuilder.toBlocks(markdown); + assertEquals(2, blocks.size()); + + Inline linkInline = blocks.get(0).getParagraph().getContent(0); + assertTrue(linkInline.hasLink()); + assertEquals("https://example.com/page", linkInline.getLink().getDestination()); + assertEquals("a title", linkInline.getLink().getTitle()); + + Inline imageInline = blocks.get(1).getParagraph().getContent(0); + assertTrue(imageInline.hasImage()); + assertEquals("https://example.com/img.png", imageInline.getImage().getDestination()); + assertEquals("alt text", imageInline.getImage().getAlt()); + assertEquals("", imageInline.getImage().getTitle(), "no title given in the markdown -> unset, not null-ish garbage"); + } + + @Test + void imageAltTextKeepsCodeSpansAndLineBreaks() { + String markdown = "![see `config.json` for\ndetails](img.png)\n"; + + List blocks = MarkdownBlockTreeBuilder.toBlocks(markdown); + Inline imageInline = blocks.get(0).getParagraph().getContent(0); + assertTrue(imageInline.hasImage()); + assertEquals("see config.json for details", imageInline.getImage().getAlt(), + "code-span literals and soft breaks inside alt must not be dropped"); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformerTest.java new file mode 100644 index 0000000000..60aad02f58 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformerTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.metadata.XMPRights; + +/** + * Verifies {@link CreativeCommonsDocumentTransformer} against a synthetic XMP rights + * metadata set (this module's test-document corpus has no dedicated CC-licensed fixture), + * and confirms it does not fire on documents with no rights-related metadata at all. + */ +class CreativeCommonsDocumentTransformerTest { + + @Test + void mapsRightsAndLicenseUrlAndTagsTheRest() { + Metadata metadata = new Metadata(); + metadata.set(XMPRights.MARKED, "True"); + metadata.set(XMPRights.WEB_STATEMENT, "https://creativecommons.org/licenses/by/4.0/"); + metadata.set(TikaCoreProperties.RIGHTS, "Creative Commons Attribution 4.0"); + + CreativeCommonsDocumentTransformer transformer = new CreativeCommonsDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertEquals("https://creativecommons.org/licenses/by/4.0/", builder.getMetadata().getLicenseUrl()); + assertEquals("Creative Commons Attribution 4.0", builder.getMetadata().getRights()); + assertTrue(builder.getExtraCount() > 0); + } + + @Test + void doesNotApplyToDocumentsWithNoRightsMetadata() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "text/plain"); + + CreativeCommonsDocumentTransformer transformer = new CreativeCommonsDocumentTransformer(); + assertFalse(transformer.appliesTo(metadata)); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/DocumentTransformersTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/DocumentTransformersTest.java new file mode 100644 index 0000000000..ab0165b68e --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/DocumentTransformersTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.metadata.XMPRights; + +class DocumentTransformersTest { + + /** + * A plain-text document (no format transformer applies) that also happens to carry + * Creative Commons rights metadata must still get the universal Dublin-Core-ish fields + * (title, authors, ...) mapped by the generic fallback. Before the fix, the + * cross-cutting {@link CreativeCommonsDocumentTransformer} matching on its own set + * {@code matched = true} and suppressed the fallback entirely, silently dropping title/ + * author/description/keywords/languages/dates for any generic document that happens to + * carry rights metadata. + */ + @Test + void crossCuttingTransformerDoesNotSuppressGenericFallback() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "text/plain"); + metadata.set(TikaCoreProperties.TITLE, "A Generic Document"); + metadata.add(TikaCoreProperties.CREATOR, "Jane Doe"); + metadata.set(XMPRights.MARKED, "True"); + + Document.Builder builder = Document.newBuilder(); + DocumentTransformers.defaults().transform(metadata, builder); + + assertEquals("A Generic Document", builder.getMetadata().getTitle(), + "generic fallback must still run alongside a matching cross-cutting transformer"); + assertTrue(builder.getMetadata().getAuthorsList().contains("Jane Doe")); + // the cross-cutting transformer's own mapping must also still apply + assertTrue(builder.getExtraList().stream().anyMatch(f -> f.getKey().equals(XMPRights.MARKED.getName()))); + } + + /** + * When two transformers both apply to the same document (e.g. a PDF that also carries + * Creative Commons rights metadata -- an explicitly supported, documented scenario), each + * transformer must not re-tag a key the other one already mapped to a typed field. Every + * key should appear in the tagged tail at most once. + */ + @Test + void doesNotDuplicateTaggedTailWhenMultipleTransformersApply() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "application/pdf"); + metadata.set(TikaCoreProperties.TITLE, "Report"); + metadata.set(XMPRights.MARKED, "True"); + + Document.Builder builder = Document.newBuilder(); + DocumentTransformers.defaults().transform(metadata, builder); + + long titleTagCount = builder.getExtraList().stream() + .filter(f -> f.getKey().equals(TikaCoreProperties.TITLE.getName())) + .count(); + assertEquals(0, titleTagCount, + "title is a typed field (already on document.metadata.title); it must not also " + + "be duplicated into the tagged tail by a second transformer"); + + long markedTagCount = builder.getExtraList().stream() + .filter(f -> f.getKey().equals(XMPRights.MARKED.getName())) + .count(); + assertEquals(1, markedTagCount, "every tail key should appear at most once"); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformerTest.java new file mode 100644 index 0000000000..06d3932c3c --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformerTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Verifies {@link EpubDocumentTransformer} against a real EPUB fixture: typed common + * fields land on {@link org.apache.tika.grpc.v1.DocumentMetadata}, and the many + * EPUB-specific properties (epub:*, dc:identifier, OPF/NAV structural fields, ...) land + * in the tagged tail. + */ +class EpubDocumentTransformerTest extends TikaTest { + + @Test + void transformsRealEpubFixture() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testEPUB.epub")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + EpubDocumentTransformer transformer = new EpubDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertFalse(builder.getMetadata().getTitle().isBlank()); + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformerTest.java new file mode 100644 index 0000000000..26a3486053 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformerTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Verifies {@link HtmlDocumentTransformer} against a real HTML fixture: typed common + * fields land on {@link org.apache.tika.grpc.v1.DocumentMetadata}, and the many + * HTML-specific properties (html:meta:*, Open Graph, Twitter Card, ...) land in the + * tagged tail. + */ +class HtmlDocumentTransformerTest extends TikaTest { + + @Test + void transformsRealHtmlFixture() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testHTML_metadata.html")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + HtmlDocumentTransformer transformer = new HtmlDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertFalse(builder.getMetadata().getTitle().isBlank()); + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformerTest.java new file mode 100644 index 0000000000..02dc464bb9 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformerTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.image.JpegParser; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Verifies {@link ImageDocumentTransformer} against a real JPEG fixture carrying EXIF, GPS, + * and IPTC metadata. Uses {@link JpegParser} directly rather than the auto-detecting parser: + * on a machine with GDAL command-line tools installed, MIME-based auto-detection can route + * {@code image/jpeg} to the (unrelated) GDAL raster parser instead, which does not extract + * EXIF/TIFF metadata at all. Targeting the parser this transformer actually cares about keeps + * the test deterministic regardless of what else is on the host. + */ +class ImageDocumentTransformerTest extends TikaTest { + + @Test + void mapsTiffDimensionsAndTagsTheRest() throws Exception { + Metadata metadata = new Metadata(); + // Calling JpegParser directly skips detection, so set Content-Type the way + // AutoDetectParser normally would before delegating. + metadata.set(Metadata.CONTENT_TYPE, "image/jpeg"); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testJPEG_EXIF.jpg")) { + new JpegParser().parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + ImageDocumentTransformer transformer = new ImageDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertTrue(builder.getMetadata().getWidth() > 0); + assertTrue(builder.getMetadata().getHeight() > 0); + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformerTest.java new file mode 100644 index 0000000000..62de41b102 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformerTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +class OfficeDocumentTransformerTest extends TikaTest { + + @Test + void mapsTypedFieldsAndTagsTheRest() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testWORD.doc")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + OfficeDocumentTransformer transformer = new OfficeDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata), "should apply to application/msword"); + + Document.Builder builder = Document.newBuilder(); + java.util.Set consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + DocumentMetadata meta = builder.getMetadata(); + assertFalse(meta.getTitle().isBlank(), "title should be populated"); + assertEquals("Sample Word Document", meta.getTitle()); + assertEquals(1, meta.getAuthorsCount()); + assertEquals("Keith Bennett", meta.getAuthors(0)); + assertTrue(meta.hasCreated(), "created should be populated"); + assertTrue(meta.hasModified(), "modified should be populated"); + assertEquals(2, meta.getPageCount()); + assertEquals(122L, meta.getWordCount()); + assertEquals(699L, meta.getCharacterCount()); + + // Everything not mapped to a typed field (revision, extended properties, + // last-author, comment info, hidden-text flag, etc.) lands in the tagged tail. + assertTrue(builder.getExtraCount() > 0, "tagged tail should be populated"); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformerTest.java new file mode 100644 index 0000000000..e88db845ff --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformerTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Verifies {@link PdfDocumentTransformer} against the real {@code testPDF.pdf} fixture. + */ +class PdfDocumentTransformerTest extends TikaTest { + + @Test + void mapsTypedFieldsAndTagsTheRest() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testPDF.pdf")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + PdfDocumentTransformer transformer = new PdfDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertEquals("Apache Tika - Apache Tika", builder.getMetadata().getTitle()); + assertEquals(1, builder.getMetadata().getPageCount()); + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformerTest.java new file mode 100644 index 0000000000..8a0506e650 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformerTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +class RtfDocumentTransformerTest extends TikaTest { + + @Test + void mapsCommonFieldsAndTagsTheRest() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testRTF.rtf")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + RtfDocumentTransformer transformer = new RtfDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + // testRTF.rtf carries a title, an author, a creation date and word/page/character + // counts. Assert on what this real fixture actually has, not on every field the + // transformer is capable of mapping. + assertEquals("Test d’indexation Word", builder.getMetadata().getTitle()); + assertEquals(1, builder.getMetadata().getAuthorsCount()); + assertEquals("Bibliotheque", builder.getMetadata().getAuthors(0)); + assertTrue(builder.getMetadata().hasCreated()); + assertEquals(1, builder.getMetadata().getPageCount()); + assertEquals(3, builder.getMetadata().getWordCount()); + assertEquals(21, builder.getMetadata().getCharacterCount()); + + // RTF/OOXML-specific properties (e.g. extended-properties:Company) are not typed + // fields; they must still show up in the tagged tail. + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformerTest.java new file mode 100644 index 0000000000..3f3e16e4be --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformerTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Verifies {@link WarcDocumentTransformer} against a real WARC fixture: it applies to + * WARC content types and routes the WARC/HTTP header fields into the tagged tail. + */ +class WarcDocumentTransformerTest extends TikaTest { + + @Test + void appliesToWarcAndTagsTheTail() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testWARC_multiple.warc")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + WarcDocumentTransformer transformer = new WarcDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/resources/test-documents/sample.txt b/tika-grpc-mapper/src/test/resources/test-documents/sample.txt new file mode 100644 index 0000000000..1461a4ea78 --- /dev/null +++ b/tika-grpc-mapper/src/test/resources/test-documents/sample.txt @@ -0,0 +1 @@ +Sample plain text document for generic metadata mapping tests. diff --git a/tika-grpc/README.md b/tika-grpc/README.md index 2ebea44c00..2268b99589 100644 --- a/tika-grpc/README.md +++ b/tika-grpc/README.md @@ -1,8 +1,8 @@ -# Tika Pipes GRPC Server +# Tika Pipes gRPC Server -The following is the Tika Pipes GRPC Server. - -This server will manage a pool of Tika Pipes clients. +The Tika Pipes gRPC server exposes fetcher and iterator management and document +fetch-and-parse over gRPC. It runs a pool of Tika Pipes worker processes and routes +requests through the configured fetchers. * Tika Pipes Fetcher CRUD operations * Create @@ -19,6 +19,59 @@ This server will manage a pool of Tika Pipes clients. > the config. See the > [Tika gRPC security configuration docs](../docs/modules/ROOT/pages/using-tika/grpc/index.adoc). +## Typed parse output + +Parse results are returned as `org.apache.tika.grpc.v1.Document` on +`FetchAndParseReply.document`. The previous `FetchAndParseReply.fields` +(`map`) has been removed. + +Rather than one proto message per source format, `Document` models content and +metadata by concern: + +1. **Content tree**: `markdown` (the authoritative render) and `blocks` — the same + content parsed into a structured tree of headings/paragraphs/lists/tables/code + blocks/inline runs. This is format-agnostic: every Tika parser's output reaches + this shape via markdown, so content handling does not vary per format. +2. **Typed metadata**: a small, bounded set of common fields on `DocumentMetadata` + (title, authors, description, keywords, languages, dates, counts, dimensions, + rights) — the cross-format facts every consumer wants, typed. +3. **Tagged tail**: `extra` (`repeated MetadataField`) carries everything + format-specific (PDF permissions, EXIF/GPS, OOXML core properties, …), typed where + Tika's own `Property` declares a type and a string otherwise — never guessed. This + is the lossless catch-all; nothing is dropped. +4. **`format_category`**: a cheap routing hint, not mutually exclusive with `extra` — + a PDF can still carry Creative Commons rights metadata, for example. +5. **`embedded`**: recursive — a PDF with an embedded image is one `Document` whose + `embedded` list holds a fully-typed child `Document` for the image. + +Supporting artifacts live in sibling Maven modules (also listed in `tika-bom`): + +| Module | Role | +|--------|------| +| `tika-grpc-api` | Protobuf definitions (`org.apache.tika.grpc.v1`: `document.proto`), generated Java stubs, bundled `FileDescriptorSet` under `META-INF/` | +| `tika-grpc-mapper` | Maps Tika `Metadata` to `Document` via per-format `DocumentTransformer`s (code, not schema) plus a format-agnostic markdown-to-block-tree builder | +| `tika-grpc` | gRPC service implementation (this module) | + +Client migration (summary): + +| Before | After | +|--------|-------| +| `FetchAndParseReply.fields["content"]` | `document.markdown` | +| `FetchAndParseReply.fields["Some-Key"]` | `document.extra` (find by key; typed by Tika's declared `Property` type, string otherwise) | +| `FetchAndParseReply.fields["Content-Type"]` | `document.content_type` | +| Flat string keys for PDF/Office/etc. | `document.metadata` (common fields) plus `document.extra` (format-specific) | +| Ad hoc title/author strings | `document.metadata.title` / `document.metadata.authors` | + +See [tika-grpc-api/README.md](../tika-grpc-api/README.md) for the full shape and +[tika-grpc-mapper/docs/EXTENSIONS.md](../tika-grpc-mapper/docs/EXTENSIONS.md) for how +to add a new format. Transformer tests live under `tika-grpc-mapper/src/test/java`. + +Build the API and mapper with the rest of the reactor: + +```bash +./mvnw -pl tika-grpc-api,tika-grpc-mapper,tika-grpc test +``` + ## Distribution and Maven Artifact **tika-grpc is designed to be run via Docker — it is not a standalone runnable artifact published to Maven Central.** diff --git a/tika-grpc/pom.xml b/tika-grpc/pom.xml index 7d021a11e6..f791489bfd 100644 --- a/tika-grpc/pom.xml +++ b/tika-grpc/pom.xml @@ -135,6 +135,16 @@ j2objc-annotations ${j2objc-annotations.version} + + ${project.groupId} + tika-grpc-api + ${project.version} + + + ${project.groupId} + tika-grpc-mapper + ${project.version} + org.apache.tika tika-async-cli @@ -380,6 +390,9 @@ com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} grpc-java io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + + ${project.basedir}/../tika-grpc-api/src/main/proto + diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java index 7213217edb..d359ec1504 100644 --- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java +++ b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java @@ -19,6 +19,8 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -59,7 +61,10 @@ import org.apache.tika.config.loader.TikaJsonConfig; import org.apache.tika.exception.TikaConfigException; import org.apache.tika.exception.TikaException; +import org.apache.tika.grpc.mapper.DocumentBuilder; +import org.apache.tika.grpc.v1.Document; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; import org.apache.tika.pipes.api.FetchEmitTuple; import org.apache.tika.pipes.api.PipesResult; @@ -291,6 +296,7 @@ private void fetchAndParseImpl(FetchAndParseRequest request, } Metadata tikaMetadata = new Metadata(); + long parseStart = System.nanoTime(); try { ParseContext parseContext = new ParseContext(); String additionalFetchConfigJson = request.getAdditionalFetchConfigJson(); @@ -312,16 +318,21 @@ private void fetchAndParseImpl(FetchAndParseRequest request, if (pipesResult.status().equals(PipesResult.RESULT_STATUS.FETCH_EXCEPTION)) { fetchReplyBuilder.setErrorMessage(pipesResult.message()); } + long parseTimeMs = (System.nanoTime() - parseStart) / 1_000_000L; + List metadataList = new ArrayList<>(); if (pipesResult.emitData() != null && pipesResult.emitData().getMetadataList() != null) { - for (Metadata metadata : pipesResult.emitData().getMetadataList()) { - for (String name : metadata.names()) { - String value = metadata.get(name); - if (value != null) { - fetchReplyBuilder.putFields(name, value); - } - } - } + metadataList.addAll(pipesResult.emitData().getMetadataList()); } + Metadata primary = metadataList.isEmpty() ? tikaMetadata : metadataList.get(0); + String body = primary.get(TikaCoreProperties.TIKA_CONTENT); + Document document = DocumentBuilder.build( + primary, + metadataList, + body, + request.getFetchKey(), + pipesResult.status().name(), + parseTimeMs); + fetchReplyBuilder.setDocument(document); responseObserver.onNext(fetchReplyBuilder.build()); } catch (IOException e) { throw new RuntimeException(e); diff --git a/tika-grpc/src/main/proto/tika.proto b/tika-grpc/src/main/proto/tika.proto index 7d365f95d1..8f5b1423d3 100644 --- a/tika-grpc/src/main/proto/tika.proto +++ b/tika-grpc/src/main/proto/tika.proto @@ -14,6 +14,8 @@ syntax = "proto3"; package tika; +import "org/apache/tika/grpc/v1/document.proto"; + option java_multiple_files = true; option java_package = "org.apache.tika"; option java_outer_classname = "TikaProto"; @@ -68,8 +70,8 @@ service Tika { Get a pipes iterator's data from the iterator store. */ rpc GetPipesIterator(GetPipesIteratorRequest) returns (GetPipesIteratorReply) {} - /* - Delete a pipes iterator from the iterator store. + /* + Delete a pipes iterator from the iterator store. */ rpc DeletePipesIterator(DeletePipesIteratorRequest) returns (DeletePipesIteratorReply) {} } @@ -111,12 +113,14 @@ message FetchAndParseRequest { message FetchAndParseReply { // Echoes the fetch_key that was sent in the request. string fetch_key = 1; - // Metadata fields from the parse output. - map fields = 2; + reserved 2; + reserved "fields"; // The status from the message. See javadoc for org.apache.tika.pipes.PipesResult.STATUS for the list of status. string status = 3; // If there was an error, this will contain the error message. string error_message = 4; + // Strongly-typed parse output from Tika. + org.apache.tika.grpc.v1.Document document = 5; } message DeleteFetcherRequest { diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java index 508c672e73..8aab23df83 100644 --- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java +++ b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java @@ -179,7 +179,7 @@ static void cleanConfig() throws Exception { void testHttpFetchScenario() throws Exception { AtomicInteger numParsed = new AtomicInteger(); AtomicInteger numErrors = new AtomicInteger(); - Map> result = Collections.synchronizedMap(new HashMap<>()); + Map result = Collections.synchronizedMap(new HashMap<>()); List errorMessages = Collections.synchronizedList(new ArrayList<>()); StreamObserver responseObserver = new StreamObserver<>() { @Override @@ -198,7 +198,10 @@ public void onNext(FetchAndParseReply fetchAndParseReply) { LOGGER.error(errorMsg); } else { numParsed.incrementAndGet(); - result.put(fetchAndParseReply.getFetchKey(), fetchAndParseReply.getFieldsMap()); + if (fetchAndParseReply.hasDocument()) { + result.put(fetchAndParseReply.getFetchKey(), + fetchAndParseReply.getDocument().getMarkdown()); + } } } diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java index 6de9b01bda..9b29906075 100644 --- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java +++ b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java @@ -420,7 +420,8 @@ public void testBiStream(Resources resources) throws Exception { StreamObserver replyStreamObserver = new StreamObserver<>() { @Override public void onNext(FetchAndParseReply fetchAndParseReply) { - LOG.debug("Fetched {} with metadata {}", fetchAndParseReply.getFetchKey(), fetchAndParseReply.getFieldsMap()); + LOG.debug("Fetched {} with document present={}", + fetchAndParseReply.getFetchKey(), fetchAndParseReply.hasDocument()); if (PipesResult.RESULT_STATUS.FETCH_EXCEPTION.name().equals(fetchAndParseReply.getStatus())) { errors.add(fetchAndParseReply); } else { From 106cb74234f748f3fc3b30f59bd55e7a9b617fe8 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 14:35:11 -0400 Subject: [PATCH 2/2] TIKA-4766: address review feedback on the typed Document contract E2E coverage (the review's main ask): HandlerTypeTest now exercises the typed contract through the live gRPC server -- typed DocumentMetadata fields, tagged-tail typing (pdf:encrypted arrives as a boolean, not a string), the block tree, format_category, and embedded recursion for a container docx. New fixtures: testPDF.pdf, test_recursive_embedded.docx. Bug found by the new e2e tests: Property.get() only sees property constants whose declaring class is initialized, and the parse runs in a forked JVM -- so tagged-tail typing silently degraded to strings in the server. MetadataTagger now force-initializes the tika-core property vocabularies so tagging is deterministic. Also fixed while here: the e2e configs now opt in to allowComponentManagement/allowPerRequestConfig, without which every e2e test dies with PERMISSION_DENIED since the TIKA-4764 security gating. DocumentBuilder: the primary==null branch is now reachable -- the server passes null when the pipes result carried no metadata instead of an empty Metadata that produced a success-shaped Document on fetch failures. Status still derives from the pipes status (EMPTY_OUTPUT with no metadata is a success, not an error). Dedup: the common field block (title/description/authors/keywords/ languages/created/modified) now lives in TransformSupport.mapCommonFields, used by six transformers; Office/RTF keep meta:keyword as their keyword source via an overload. RTF gains the dc:language mapping it had drifted away from. Image stays bespoke on purpose (IPTC/EXIF sources). Renamed ParseStatus.parse_time_ms to fetch_parse_time_ms: the timer wraps the whole pipesClient.process() round trip, and fetch+parse are not separable server-side. --- .../apache/tika/pipes/ExternalTestBase.java | 3 +- .../pipes/filesystem/HandlerTypeTest.java | 206 ++++++++++++++++-- .../test/resources/test-fixtures/testPDF.pdf | Bin 0 -> 34824 bytes .../test_recursive_embedded.docx | Bin 0 -> 27082 bytes .../tika-config-ignite-handlertype.json | 4 + .../resources/tika-config-ignite-local.json | 3 + .../src/test/resources/tika-config.json | 3 + .../org/apache/tika/grpc/v1/document.proto | 5 +- .../tika/grpc/mapper/DocumentBuilder.java | 31 ++- .../transform/EpubDocumentTransformer.java | 12 +- .../transform/GenericDocumentTransformer.java | 12 +- .../transform/HtmlDocumentTransformer.java | 12 +- .../transform/ImageDocumentTransformer.java | 5 +- .../grpc/mapper/transform/MetadataTagger.java | 64 ++++++ .../transform/OfficeDocumentTransformer.java | 10 +- .../transform/PdfDocumentTransformer.java | 9 +- .../transform/RtfDocumentTransformer.java | 9 +- .../mapper/transform/TransformSupport.java | 29 +++ .../tika/grpc/mapper/DocumentBuilderTest.java | 19 +- .../tika/grpc/mapper/ParseFixtureSupport.java | 4 +- .../tika/pipes/grpc/TikaGrpcServerImpl.java | 14 +- 21 files changed, 354 insertions(+), 100 deletions(-) create mode 100644 tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/testPDF.pdf create mode 100644 tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/test_recursive_embedded.docx diff --git a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java index d0d135309d..f9120749a3 100644 --- a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java +++ b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java @@ -242,7 +242,8 @@ private static void loadGovdocs1() throws IOException, InterruptedException { public static void copyTestFixtures() throws IOException { Path targetDir = TEST_FOLDER.toPath(); Files.createDirectories(targetDir); - String[] fixtures = {"sample.txt", "sample.html", "sample.csv", "sample.xml"}; + String[] fixtures = {"sample.txt", "sample.html", "sample.csv", "sample.xml", + "testPDF.pdf", "test_recursive_embedded.docx"}; for (String fixture : fixtures) { URL resource = ExternalTestBase.class.getClassLoader() .getResource("test-fixtures/" + fixture); diff --git a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java index 18ca0c4178..b72282c664 100644 --- a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java +++ b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java @@ -44,18 +44,27 @@ import org.apache.tika.SaveFetcherReply; import org.apache.tika.SaveFetcherRequest; import org.apache.tika.TikaGrpc; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.FormatCategory; +import org.apache.tika.grpc.v1.MetadataField; +import org.apache.tika.grpc.v1.MetadataValue; +import org.apache.tika.grpc.v1.ParseStatus; import org.apache.tika.pipes.ExternalTestBase; import org.apache.tika.pipes.fetcher.fs.FileSystemFetcherConfig; /** - * Tests per-request ParseContext configuration via FetchAndParseRequest.parse_context_json. + * End-to-end tests against a live tika-grpc server (real gRPC wire serialization, real + * forked PipesServer, real fetcher plumbing). + * + * Covers per-request ParseContext configuration via + * FetchAndParseRequest.parse_context_json (e.g. + * {"basic-content-handler-factory": {"type": "HTML"}}), and the typed + * FetchAndParseReply.document contract: DocumentMetadata typed fields, the tagged + * `extra` tail (typed where Tika declares a type), the structured `blocks` tree, + * `format_category`, and `embedded` recursion for container formats. * * Uses the Ignite ConfigStore so that fetchers registered via saveFetcher are visible * to both the gRPC server JVM and the forked PipesServer JVM. - * - * Verifies that clients can override any parse context component on a per-request basis - * by providing a JSON object with component names as keys. - * Example: {"basic-content-handler-factory": {"type": "HTML"}} */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @Slf4j @@ -285,22 +294,35 @@ private static boolean isParentProcess(long pid) { return false; } + private void registerFileSystemFetcher(TikaGrpc.TikaBlockingStub blockingStub, String fetcherId) + throws Exception { + FileSystemFetcherConfig config = new FileSystemFetcherConfig(); + config.setBasePath(TEST_FOLDER.getAbsolutePath()); + + SaveFetcherReply saveReply = blockingStub.saveFetcher(SaveFetcherRequest.newBuilder() + .setFetcherId(fetcherId) + .setFetcherType("file-system-fetcher") + .setFetcherConfigJson(ExternalTestBase.OBJECT_MAPPER.writeValueAsString(config)) + .build()); + log.info("Fetcher created: {}", saveReply.getFetcherId()); + } + + private static MetadataField findExtra(Document document, String key) { + return document.getExtraList().stream() + .filter(field -> field.getKey().equals(key)) + .findFirst() + .orElseThrow(() -> new AssertionError( + "expected document.extra to contain key '" + key + "', got: " + + document.getExtraList().stream().map(MetadataField::getKey).toList())); + } + @Test void testParseContextJson() throws Exception { String fetcherId = "handlerTypeFetcher"; ManagedChannel channel = getManagedChannel(); try { TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel); - - FileSystemFetcherConfig config = new FileSystemFetcherConfig(); - config.setBasePath(TEST_FOLDER.getAbsolutePath()); - - SaveFetcherReply saveReply = blockingStub.saveFetcher(SaveFetcherRequest.newBuilder() - .setFetcherId(fetcherId) - .setFetcherType("file-system-fetcher") - .setFetcherConfigJson(ExternalTestBase.OBJECT_MAPPER.writeValueAsString(config)) - .build()); - log.info("Fetcher created: {}", saveReply.getFetcherId()); + registerFileSystemFetcher(blockingStub, fetcherId); // Parse sample.html requesting HTML output FetchAndParseReply htmlReply = blockingStub.fetchAndParse(FetchAndParseRequest.newBuilder() @@ -342,15 +364,155 @@ void testParseContextJson() throws Exception { "HTML and TEXT outputs should differ for the same document"); } finally { - channel.shutdown(); - try { - if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { - channel.shutdownNow(); - } - } catch (InterruptedException e) { + shutdownChannel(channel); + } + } + + /** + * Exercises the typed Document contract end-to-end for a PDF: typed + * DocumentMetadata fields, the tagged `extra` tail (a boolean-typed key and a + * string-typed key), the structured block tree, format_category, and ParseStatus -- + * all through the live server and real gRPC wire serialization, not the in-process + * mapper tests. + */ + @Test + void typedDocumentContractOverLiveServerPdf() throws Exception { + String fetcherId = "typedContractPdfFetcher"; + ManagedChannel channel = getManagedChannel(); + try { + TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel); + registerFileSystemFetcher(blockingStub, fetcherId); + + FetchAndParseReply reply = blockingStub.fetchAndParse(FetchAndParseRequest.newBuilder() + .setFetcherId(fetcherId) + .setFetchKey("testPDF.pdf") + .build()); + + Assertions.assertEquals("PARSE_SUCCESS", reply.getStatus()); + Assertions.assertTrue(reply.hasDocument(), "reply should carry a typed Document"); + Document document = reply.getDocument(); + + // envelope + Assertions.assertEquals("application/pdf", document.getContentType()); + Assertions.assertEquals(FormatCategory.FORMAT_CATEGORY_PDF, document.getFormatCategory()); + Assertions.assertEquals(ParseStatus.Status.SUCCESS, document.getStatus().getStatus()); + + // typed DocumentMetadata fields + Assertions.assertEquals("Apache Tika - Apache Tika", document.getMetadata().getTitle()); + Assertions.assertEquals(java.util.List.of("Bertrand Delacr\u00e9taz"), + document.getMetadata().getAuthorsList()); + Assertions.assertEquals(1, document.getMetadata().getPageCount()); + Assertions.assertTrue(document.getMetadata().hasCreated(), + "created date should map to a typed Timestamp"); + + // content: authoritative markdown plus the structured block tree + Assertions.assertTrue(document.getMarkdown().contains("Tika"), + "markdown body should contain extracted text"); + Assertions.assertFalse(document.getBlocksList().isEmpty(), + "structured block tree should be populated"); + + // tagged tail: pdf:encrypted is declared boolean by Tika, so it must arrive + // typed as a boolean over the wire, not as a string + MetadataValue encrypted = findExtra(document, "pdf:encrypted").getValue(); + Assertions.assertEquals(MetadataValue.ValueCase.BOOLEAN, encrypted.getValueCase(), + "pdf:encrypted has a declared boolean type and must be tagged as one"); + Assertions.assertFalse(encrypted.getBoolean(), "testPDF.pdf is not encrypted"); + + // tagged tail: a text-typed key stays a string, value intact + MetadataValue creatorTool = findExtra(document, "xmp:CreatorTool").getValue(); + Assertions.assertEquals(MetadataValue.ValueCase.STRINGS, creatorTool.getValueCase()); + Assertions.assertEquals(java.util.List.of("Firefox"), + creatorTool.getStrings().getValuesList()); + } finally { + shutdownChannel(channel); + } + } + + /** + * Same contract for HTML through the live server: the typed title and a tagged tail + * key, proving the format-specific transformer ran server-side. + */ + @Test + void typedDocumentContractOverLiveServerHtml() throws Exception { + String fetcherId = "typedContractHtmlFetcher"; + ManagedChannel channel = getManagedChannel(); + try { + TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel); + registerFileSystemFetcher(blockingStub, fetcherId); + + FetchAndParseReply reply = blockingStub.fetchAndParse(FetchAndParseRequest.newBuilder() + .setFetcherId(fetcherId) + .setFetchKey("sample.html") + .build()); + + Assertions.assertEquals("PARSE_SUCCESS", reply.getStatus()); + Document document = reply.getDocument(); + + Assertions.assertEquals(FormatCategory.FORMAT_CATEGORY_HTML, document.getFormatCategory()); + Assertions.assertEquals("Sample E2E Test Document", document.getMetadata().getTitle(), + "html should map to the typed title field"); + Assertions.assertTrue(document.getMarkdown().contains("Hello from Tika")); + Assertions.assertFalse(document.getBlocksList().isEmpty()); + + // tagged tail still carries the HTML-specific keys (encoding, etc.) + MetadataValue encoding = findExtra(document, "Content-Encoding").getValue(); + Assertions.assertEquals(MetadataValue.ValueCase.STRINGS, encoding.getValueCase()); + Assertions.assertFalse(encoding.getStrings().getValuesList().isEmpty()); + } finally { + shutdownChannel(channel); + } + } + + /** + * Container formats must recurse: an Office document with embedded resources should + * come back with each embedded child as its own fully typed Document, through the + * live server. + */ + @Test + void embeddedDocumentsRecurseOverLiveServer() throws Exception { + String fetcherId = "embeddedDocsFetcher"; + ManagedChannel channel = getManagedChannel(); + try { + TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel); + registerFileSystemFetcher(blockingStub, fetcherId); + + FetchAndParseReply reply = blockingStub.fetchAndParse(FetchAndParseRequest.newBuilder() + .setFetcherId(fetcherId) + .setFetchKey("test_recursive_embedded.docx") + .build()); + + Assertions.assertEquals("PARSE_SUCCESS", reply.getStatus()); + Document document = reply.getDocument(); + + Assertions.assertEquals(FormatCategory.FORMAT_CATEGORY_OFFICE, document.getFormatCategory()); + // the container's own typed metadata (docProps/core.xml carries dcterms dates) + Assertions.assertTrue(document.getMetadata().hasCreated()); + Assertions.assertTrue(document.getMetadata().hasModified()); + + Assertions.assertTrue(document.getEmbeddedCount() > 0, + "container format should recurse into embedded children"); + for (Document child : document.getEmbeddedList()) { + Assertions.assertFalse(child.getContentType().isEmpty(), + "every embedded child should carry a detected content type"); + } + Assertions.assertTrue( + document.getEmbeddedList().stream() + .anyMatch(child -> !child.getOrigin().getFilename().isEmpty()), + "embedded children should carry their resource names in origin.filename"); + } finally { + shutdownChannel(channel); + } + } + + private static void shutdownChannel(ManagedChannel channel) { + channel.shutdown(); + try { + if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { channel.shutdownNow(); - Thread.currentThread().interrupt(); } + } catch (InterruptedException e) { + channel.shutdownNow(); + Thread.currentThread().interrupt(); } } } diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/testPDF.pdf b/tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/testPDF.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1f1bcff6fe90df04656b43f98640d863a65f8660 GIT binary patch literal 34824 zcmagkQ<No5yC~|iZL7<+ZQHhO+g6ut+je!?wrzX&_hYPe&c4_;nHiBW--yUD-^?*0 z9uj$BQ5ptXCMc4jqx++a(ucg+p%ExX0(t^FLrW-bZUQ=K6I(N9a{}hSRf+_3q88T9 zCXRp4)&|Zd!X`#`#wG;3yiiWgjwS{+Q10tg$`H!rAVxl|+CreI-XZ(|3{4<cP$ss< z|7r2}>Yu>>MEmE=$n?*>f36t*SC`5j_9g^$@&;!AIy#!zIuo${yL&|wCp#BMBNHbA zrhium+1Wb(z3)Wu&yfF)PsYU9!a&f@9ZHM-Zv_hn3jsSbqs~7={7*ms{O-R~lXtW; zQZjKS(E8h%uqc5J0iC*>p{0qDGr>O-7qwv^VEI>6Uf%x|;Ge4h2KdjjVgFYvx_=M* zzai+9TnwH6H4#xeN1MOl{vQ9;f`Nce@b3r-{y!sxqWhnzvHm;U|Laj#`)Ju|izOa= zeMfpa179zA^iF23U;sl))0yf6J}}ZDiD)&ED9R{a-F?4*^;9{HaVBU~RcK(+Log(# zwuu=FXZhiLAKb<Tje*<25he?bU*CHBz0M3C?Ah`4qz!IO^e_JCUc>bttoTo!-L4qJ z?D=w>zVA$A4^Tdh_F~bdR`YH8aD3Lffft_i9{)`2ZKV$G_%bGeJ$CBB?R?#)4tDNn zrZHol=uC^Ktlsoo14d^v$Hw0~)Zc7F`?GsoNk=pFZRssi_%A6zQ3TTid#~5(527Sd zY|j9^Q2BELvGZ79XFD<V8T1A=CcSX3y}jHF-8>u`(Q{6<danYzGo{OWYtY_x0)!{Y z_9bb})peT8Cbh2Bm?4<KxQ=g&B78yIu6)q|xBP+pgI5z@nit~+odd$~3*k(##X)$o zs1%`z#X@-Uu(ZHu(N%a-nH1Y`>~Hy!EGbLtO$34I0FAYno!E3v>24W>ufQ}yDT3jZ zzQB}1DY~O79BGkxT~1%ZO>)Z5$1Qof^>Cc)7s<uKcg}W5If{<y>Wwa_Q~;F{vj+@7 z4a_tt6HzI+8ra3YV6|TY0D;<_LHo(h^Wo#JB&@P6zz1qzZbKAR3EmKkPzI5(UH?;I zG7eWrVvL@KBmi-^+Z;4UFQk94P(}mP2{@!bYV@=NaTYH&IYiPt!jit0m9=);vxGJi za1z!43ov&SDpZZd7iD)62n4&ay#}oDOpuL$LMJ(dz%@Y46I_O4RnUF%!MHroLO*+* z&|Jx{u5*ZZvP`LZL3cG0(^BK73NSKA1b=aJfM&5nAOI8F{HZU9gm_6t7xnPB$~KZi zQjD90a5Lb;kz*CVu1GCUzB2^1r&|K_aAXw^tg|?T$z;NIAx$uHr;Dg*GBfT{{l$nq zS%Q7pevEqX7(>c2n%PFmuNAj*B9;_;G2}dd&rTX=r5OBpAJ-)_j_YTsQRF&KeF+Bw zU9D~p0(!l@T<)IlVkrSEx`BMUS)8$=yp&(y6fUj>b6d2$PasT~;(P)q0+ApTO5^oe zTA<{iff7`~s9APMDE`kETRjUrfW;v%K*BZzkRvlkb%DebL9UU!!Op)(fNgRg>bpch zn*UTTBhubAWS&VBjo7tj-q>VA;awU@1@T4TU@*brYGa|)t`tnp9?3@Hy<FPA;5vPB zsuViDQZ>0eO;F9o@!>N^8+;jM;;THg^8Ku)%BaC%M&8_lKF*Ee^Wpn$UqCkt9=xz$ zeIh?PfVXL2q3fm8@Bw)i5eI6l@P4<^#(IUt$Gy0|yWeyKCt_==cbbeI%$UK4?`R1l z=q=ufrM*AA0N%EEe|f;yKC@L^(DBt`-^z~Q-W;BA1!#`)6~8I4_<4nf!lf8+tlpip zmYw!lOvLqfIoJW%O+1*nfPK5K9K1iwT!7K!F8e!DFME7j$>Z%K#_e0HIpl0Tz($tL zsN*Ja0?|FY1o?|W(1i9m>Dn^XVT0@=m^Cb&LFA)BcM2F^Ch8XrHjEg+1BOkKz|u7F zFNRC8Ac!(8mPMM?feDUCaAEX-?vH{Asz3y{%Ig~Nj3%iM-gRK=zQf#+6L4E|WSxQ< z)B+LoIV{*arIK8Of*A7YP#e35rfq`AI*Rhw>8f5Q<~KXhzJcJ9K>*9TuzGf1!hf*r z0_ex+ccfNBG}1qe6{GUYFAoSshitFF+-V33V<=&yfg1QhL0|yG7k2ADi|_{nk<GsS z9t24xQVrXK?~hv%H;}Y{x<oKQ3lMvb2&Hf7w+2=?jZRrS%8jHiJviJgStAS+P|nNJ zo)(JcxbT=qS`^>MamXlx7gfCA*6uTR(V&0sz?AE@OlbX*En_G=d0kV8DYG?&VoU(W z*=*x?10u3F3aB+uL%>8j8cVC{mxc}xk1WW{T9L4;^j`0ju9s%uN>zhE#0Wq(4Da9= z-y1qov*S2`S!62c#kJK$?R?`zP=<#u=7<kBU?k$WXjq#iE5NYF%W~1{TAw9DBJpG_ zPh)ON$!lJ3e_WTrj85`g@r>{6PaKqZ4pxo%^Y@G7sgY_<$1_XY#0XXv#o+=^Tx&N4 z5_9$iQ%$>i^|+%HqJDB{D@jTiO>=niWJ=`9YG0W@V#Eo`(T;DlQ?}JIZdMO9sCR(n z`{R>)Mc>m$AF4j`8DbBv$=Mmx`~0GyUlXCB<_20VdmsTz*LPUP*>o<P8~aSt#`GG5 zgFf^`)f;sf6mS3tn`(FLhP4*&eO<pIR*#)j3?aThej`|$2J}KSR}(i2gD|@gI!jL0 z9M;z@#cGyJaMjYx=~W-M&JU&3X-^m?Z9tekEZTThJnB;?_F!i;jlJ_psTxoZ6$-hF zh`6ZHQ;#Oe_l`lA;9!Cq9=||Z$QWYLorBf$PVOhS0jX7NYvinj4_^gI2Cg~^yfblv z^a?!8R`hNMC<u?e9ij2GTnQ2Yob4h+NeGiN13Je#KrmS;v$&d1`1_E$gP&8&EwoX8 zMse{ix})Bkm431}mIK-`-eH?5>!>Lpbm~ZT+EM!PweIU9rLpIdV*-=!Eel)Ag$}=l z|IK__oQ;PqSdt_rrYUIfQYYGAGXL?~kaD^vL$n6yrN$-jP=b5O<Ckh8E>4V~<g{Mx z-O}AoP&2H(Yl4gGp<^o6)Jq#%80`R4=*P)SUZ|b!GTH*i14(TaZ2(ges;&*sLzMgY zq^%^$sWaFNN5Gikz0jC)tOG`x!vXf4`M#v+>%^`fc{<)SreL7*;w|-UR4IT8@+fuy zN2zx7S{CaYD^NE4#2f|#Aq1Ya?W^bpg5^&*Fss{Y`Q{&O2=D;r)Xsqok8CLj;|<`g zn{;TvV`ag5Bc90@wk-F96Vu$MuglYxSL|%!p58a4t}6}+x9-t4W5+>MIV55sy^G|f zS-f`?OG)p`55VA_%c`~anwXhM&qM3cQ1l+Q<Q-G=`nbeeOWOAd;N<QRKsF7ktcjCS zI<*9W{)`ND6V;mMnU@)OxgA|vZF!se_@Hh0Zd5+rEuRm^pVxQCVmGYvb3=!2mfmh| z?r%?99unNgNN=Og)}Ei6u+){gM9WjCc5aKfpy!m&_wTo(@3)S=byD(nWtKGU+lRq0 zGlY|G6HZfGT2xNQ82FDR-&HSSmJq~7jr>ju7!dKfBUEAbGHC<;m4(SEf-Mv<Yk5bi z@UjX1)1;;qm}j$iTR7l)mcazfL(b>GKZDz7U>`^v1(Cq>_<3F4TQV_@7!XQA+US?Z z^!lFZ@9Vdv#&E#fHQkS@Tw|tCh4dFY?aFLHfO-TrnUL`Qm@w=BupG6~k+4cK{iI7A z@@zqfOi8X2wsA-xL}mhPE(ZZwd)Yx;{j=nr40V+fD<H#NnbC*ogZ5+8#%eqW?E7QC zuQVA~#pHh@DyN|yYS@eB;TloQ0}%yX_~)q&@<LmdLpx#6w~4rk>}R?gg~bE$^=A>H z1*3eYZNz>$)Vk<zlX3cHVn`9q4lkA;g0X!Z!b;mw_y)bI9OEL438`!_z*6s%m@yrC z0VV|C>Zv=fCX14NhK8LXQXHu${Yeg>fZ%py;K`p?Mx=HTf1S%j+S?3%_IPgSYHp6j zYnD1;?nhf#|C1=VKC;)Hl^uBSU?f+MZ=@+H<y2|M_yo|)G18TP6EumwyS!Zz199U; z0Sa&T5>2pLHs)^dTS7uYoiMYB>sQh(&+fnqf{#dK#}RCPPnWQ&w~O=i?=kcY8hq<h z8WM)zl@kkpLWYz_0z2LVnJQ#G5~SS^xfgg@^qd_`5Z{inzC=iuQxucJ@(<9eikoN2 z>~SX5HEy#pW!)u#gPu~rkM--M9mcwkm~e%(qGnl*xm?rp<aaSQFhXe;`Uw}Eya^SA zQpfIwc{1ZiXp!3tDQPWx&v7F%`+`e*=2I%KatBk;u{$V_PIP;I1sD1J(*FQ~dBE*| zWo)Ujm*tO@58U0oQjeIEZk)kq6x+jeWwAWzo>U;vreir`u4Aw-<f*w)#AL!J4vE|f z#m+{a$&kqH>=~0|lmonLVm+z{3+e`bAh}T8OMyh3AWn-g)25wC#lw~{uWI|_3UX{# zi0Q;}5k#oX8A|~<3d-5ej2A-f^Ag>reWs^LBhm*(?AD2`Q`z#nR&?n1?7_>$rXzkj z{d|6<%J)U5t}1ZVF{}0$f&Fic1mVWJ#~DTwgZ_nf2QELPwsCK7dTiF83%YK_J!5Yp zo8BGhI6Ci#+@M12)f`cF<NFf{*87WvgNcQQuhGFUKKR<)>)HmX^O)3WAozi~v#O6A z%D38^oAs?lkyxfjCj%vMvwpu1SGON`Kb=XqAH**7w7nGQ?~fh7t~tbdZ$;|p;|vuu zP$r$Th`u!;|7%aFuba_MJN)flUTz-Uc2gf5=O=f|o5mjPG38rSgf`I)t;@G>{2jbM znkzJrA7j4BWt_qu??>aqWk~wZURK>dpKs&CyJ%RXv7VJY+dp@2GgPpM(CGSZLNCs4 zZZCRzwIX=U6tBZ4+lhL)xxSsfUfn4}BI(AW0hU^sz&{l97mmRxq;SD8)Ho3nf%6I_ z4@OP}5t#yboHTN@CTM!V!Y&h-1?e!-ID-wwwq+0zRnd+lH9z0hyd)OKxIh=)l#k7* zgw)p5>vm~YRD!{^DM;N?Q}TvA8?CavzhOgcOrXQXoo>%i0O@1K!cpqP*_B+eDTHBB z3cjM{b9g6U9CxfLD<ndmh(C$~P5Bcu26Zz(8mNaD>qm<l(gy1kqDcx4kO$R*qRuzR zi+?VQM;)1mW80H4>&&I;sZw1Rbn|;UC!;<Il{5oD-Rp;^Sad9^)*M)Xi4-0PnDkQc zbQ;qqnFXG?sjSuF;~4dZd%}_rFc*iIJ=htsVTEIXx0%B=2Qx5*O&OGO{INWxf#Foa zYBYjv8qC6B1;SYmV;~y3s%}|m-Cch9p>4B<D(Mt}$wjL&)y2jL`<$DMA{p}bR%E_( zU#eGhM+=>W24p8bVcThsX)a@>h@0*l(A*K5tpTa77BlvsfsVM47qiMiOz%v24lo8; z>uW2kVF<&;VEVdl!-NEIFt_nZ8<z;YBpnIWK*M5B1ZHT)Xhw0H@er1KJQ`rJ`ks-_ z3C3io1x7XCgdHpgF70U`p(ffiJ+Wz|9H?U=I;y#@Xs6z_80@KOqyR4tBC;C|rj_2u zegOzg1Z?mxo$inBUolrYyR5c-dZmv8>1LVgo|5)BHj%f<Uyrp)GibQR9yOGYmg>?@ ztm6z&?&uj8K*{nO|6Ddig50rW=V}KTDx?Ro_mzrDagRSDT~M80+{$L!MdY9MW!DST zcQFImpchQ**wmy%Vy6xqJGfZt=iO$*x>{=_Pe9-*%3Q(JXCiyM@0b&LSlE}+>1weg zv#HJ#A7Gh0c26p51Yb;B+nzUTRM7u6X8XOWIx^7(uZKXGW>zyCi9HIJG2%|8;eecY zckGGlCGA`{j&y10CAwEgU3!OHil3(R#zRwTFhGCKXi8hss844UN3)nR4HTxyrkY3H z4!L&Xu$j*dJ;#n=G^tHpTpr?KRtBKqZvuBF>M}UumO4LuUE$kqAR~);s=@YLXz4h< zMA@sc^+y0*8e<4!rma(2*5s9@`hlEtE}wk2)taF9nbQzah|}U=X_$gm-xU8@Qc`w# zc!1ymV#{jF%)-K_B2yrTGvAV&(`@wzQ+Qebl;XlDe+-6*TG=?gu2zBn`RhAWc+NED zr(;}-jJ%1n^r~rv!%3qNU~axP_4I?-Pz)}-p0sv0u{=)zHj|h&-Y;@B!2q9#m}w*? z0BUi2&bDf$qryXbq5TCL@xvrdYO=pT#O~6S;O$<nlw+mft$*rwy}HN>_$KCIQ1l0L z&C=xpBpyxXnzOdXg;VX#uRiWhr%HRrAQ}^s>K`g#=}~Tgq#Ge3c@%^|+tIJp%5?D_ zv`y#J$CfpHfshB4UU5#B0kIbMrG^_1HidnwJYyc8hYNUTVxrP00`~KYBwOMN4WZe! ztAZw1)5Ag5&S_gc;c9t-UDN~DmgxWl$BxSkV4g@g@b_s|bS@BqU<g*&^85;GF8>>7 zZU#b5#iC_KNk;U>&_qx)j^ZBE#C)-<lnhYarwLfg-ee9pyV%vaW+_{h){2Eq11-WE zkoa&tMVf|niYuCwz#m+SBcqVPQ~Hj!`VmOjMI>CHydowxNl(gnN4`yQjn|#_V_#cD zjLW_u`6+fh#GS-5cuYdk=MvWaw^?YS?zx^a23yH$`^U`d_eE$Tqr2_uPy4G2u&{&G z1~!5U+%QKZ>kd_Vz0P{lWF_m^F8FcvW)w@2cky$autJUONPE4Oyvc??+{M_a1VGZk z{I4-8cIwlZ8Sv#STf8^V)oN`08Ou!5iO`-6EM_dB`f-&l(8DU=Y;%(j@u59P$z)!t z(fAW=X#UF#F}JRgq(Heo&FF61;qp<}g1@?Qy;zX#zdK3}kl?{t(Q`c3v&tHqN2{v+ zm6pkI-ZH9T!Q)xXz9rmeE`R9D71&pLJuTRqbhZ;8CU+e<&|NqVDjRctAI(~xR(fTY zh_-)`3MmW{k4T2Wnbp4yny<VHWXmVhxN^2a>5Sc3rwmgWeLwcsRnXeJW(#WoMJZ>u zYjpYOzY{g)3MKCHb-Eyt7BNLGHyTi`9xlokIuT$e3r`wZ+@`BUG~%;cB1A;j4YEv@ zx*`&>ztG?>C_!0XA3=1jFUEjKZ-7l;ID#b<F0l*yGf}9<+Zj$GQr)^tNly~sa{}Wa zB02>w0}&a*frWD{vZZ1kHY<(JJkN60{(6EOdO-<_hj!!$;&#D9V`_3oGhYMZgPV$O z9<B=1{y@`YQ7+#~_F^obUVsrp49<AM(#g7s8*yLesyW>VqqBELta$e}MIBUWst-(_ zqGxWgSxz0s6vj&p#U7HT`t$pa8rm{Y?wAHoRSLmkEz9Saf7W8#sYCOIA6@x%k?l8* z(G5S*F+gm(PE#esGt?t4&!DI==hDmUFV@g9Q8VAl@ISXQ${R^cmEY+XS|<0Tw&KZv zj)vq%D^?WckQjt1@(%@rDg;`$bJz@i2{a!#U32taC*??Ri8XIG*){TH@X@qyG2ppg zK0lBQ8}ce;nrtf8n#IcJjxApsv-j`0-DKiQ_c6}K`Wk_5I-KKd%-Cxk8UuUxTZ<B_ zWo!Nyj&TdQUIHc0ns%DEJ?JKu6X~fF0lsHTcVQUTwsq7%b3-NHh9l9#Y~p5?c5~D6 z<*ef%q*Iunf6f2}!a^JIj0v#Vhx<qC-8Y6dL*|Tt3tqv}^HL}GL|b;%SAsF5Tp&o` zlk*b`+|0uHqe_e%S{o?H5Qa{r;$*lOXs9eB*Wie$W(l)NGf_snhR?l`yL-_>q`rC$ zq{`rVRC_OEsOY_!IE&vSW=t-27n&GGQn#srs;@|OWeD<GG(cMR_Gh*bklpWuG~}C` z<X6Vzeu<tGnnyfmRU8+Z2SE2|<V|;4>|c^9O;-Um8gPIF5qu5#;r`OkiZhDQGXsCc z!mDoTA$^!Rnay+IbS@Sd3!9VQEOGQY`mh-1NG5le(|J9QP@g8-c_J4EgUzRQwW40K zs`cjkSXP9*Nk97FXikxYOKPb>LrJNBx!l`eNNx3)VeG269<pc6(HO4(Bs*+i6fuJk z(El>JH5$~+1Ymb~6D`?Is-T?`m~jL>P)V}dI}wj)(D3IGdseN{-_n@Y=~a7XyG3L@ z4L!_#e}K_0v=!F;jYva8=~Ib&pR9u)qn2u}Ajcrfaa4bG*mRfG^2B*_MiMC1&0E-U zzH7*PRP*`N4Pt<;Q-35X7h%_StD)USK6~tMr1f~7>QaIPIvo#3yKOlIrN_8UE9(ES z&lKz(kn*tIe^RG}me}eq{z*Dr;jp_1YFt~7ic3Kb-EHHHYvdq`j7BctQu8!xaVNGo zK00F3Ix1@kfJRewA+dSuBYBdx+3`n+jA_@<IG>JRnGH8_tt+66u&VNpnGtDA_A|Dv zFi`a(8jq_YY($^g0RR^Xg13PeD+UJG;0N9ggw3Fu(Zy;_>*ccqDZIPs7LG<T-_oT2 zH0+Ku8DQuif$(2Ff0qZ`tcPPLayysND!JI{V~_F=tnRMZgEY;^`&$nnEXz0CyQkY^ zOpUt!F5<0UOu$2svw?YC!+|U^WvaR?;6kKnU^h_TYtik7WtwlD4a?!>t?^&38a4WF zJ^_6r$a(nuENX#g9k~~l>>BHeD^c>u+?B6%;46k6aG}VxT#IElsr|ZA1qTv*$&8xm z^1BIP2Ysb087=mE*8KEoS+3O-3++uA<Y|sF8r!d|2j`G<I<Az=$L$^jb2^dAUz_`# z1cxiMfP$ZiBHf`7bmzf&qSzt+Nl@sAW<P+7HNgP>D^v6LcH#L(gp?_KLkQ>@?kRe@ zc(|BR!!>DTkdjU6CV}yHgb+jGf@(J4SUkKeg%&??kOP)kmQ6|r33k7II6gRMciXv~ zgK?<<$kXV^x`TbpTns!>g#$Y_7M#<h5tvh`thn`LjJJTPG6?W)XpVHX$?_aa7Il~T zGNTn76gx~d^z)IBJ1dF~mtR8j?C^$r>{=^~|ETSD4k1TwzKkt{_f`|-XvWX}xQHv- zEDl_hx(w_UwH5|V(l4Vo=ZJKJ_zO{16Ivo6qhP$uZvrPc7W8%ZI=+{Rsy(F)8GgTW z5@dExlSa0MW~Fs@VGllny>DGthM%9b$Ha>l@w_3=PTHwSwI@>zRtnCJbjN--_}<>$ zmL)2CZK?B|+AEZv@dgWD;kc_&9_8L1zprOUqo^h$!M^i_5WCE!0m{+7hRbP1y`3LV zzaz)!T4|WABVP~C9@QT%daR39&dTijvydZk+WFq!QCmJ<zCC{4-Hrn6Sj$z)?3ggO z_C{_TTE7fqqsd5Sx|DO$HP29Xa4<O{$aZk*Id1qk@9p*rf5k)OLdh}+(aH5rsfi&; z7Y|yfA>4t&$o4(Nw)cV>zs-hf_HqrlO(mZiO%v=lB4kA!X~X7oVdM5<33l4x{))pC z_Vo;Q<#L322D<liB^XyvAx95I?L1(`$EU!VZlSpk`)m>SQFWBf);*<$Pd?Z|OB~fu zP`!VN{6HmNd{$@WnW=MXs|W0p3B3+LZv)`A1AqdU9Kdet*nn~?rMg{6y$oWd>MK<O z=n;K84AS)VYuitmgeM)mF>OiBE}qM=1qp)G5|3)XYy#I;k*@~U0>G1-8y>R?mn8gv z?i*b|h0+_|Gq+iN{YKGw8f2Zd-oUJBr<ov+T}a|A+R_PqLdl?8n-KN<`RKcA=_o@4 zkTxQ>X4D3l#{>90lL?yf;}Ajbto7L?F2$y;gsD=J#mU)fzl85Byv1$p@uT7L+1ktg z)xdO828+@CtfOUh86NjG1v0d#0K2-1W5#b|?c4j~Q*onGED+j}b^kQQYGX(`5>*by z>d3iFitJQS2S{%HW|?;Tf((!nCs4(1=Nw#JaG}OAn3=57J<)d-ofBYv!l?Jc#Q>ic z8?3|yDh#kZJ9x#2ZIb&8P_^9>1N44{qDP%5-VaNmMn&9ZLGCt=6{)kz5iE+J>qbTY zWiSXDI4gwSn-MScsI%UC>AKIxsj=HyVC9E`ChvI_RwSZ55{H21S8<2!ERfvAmg$42 zMc}#sCX7d*tnm(#;d5GQ`N#;~GWRv!-e1)X8}7pt8q?2oscY+`UyXw@)v%iLsqZr5 z`m43Zeid-G^O0f|bvG<~jlAk1y1fd(i}Zwz_myD>b@xlwM<9JPZ;YSTxjD(e>hDHI zw`WBNXNqZypUmL9|Dg`1PUCX7#Z`+?&qk$}Z(+ul6?C8O4Lc{Z1J}Wg7m3YY>@9LS zE;G&&x3l^FsFs-7AypaNdO2dr+Y$DkQP+`^?hMlQr_hknEfS-$po2lE+YR3UX`b8` z)N8la!&K{6novR-jZ(0n=4=Nt{j9RRetFTDS$Q#bFD$>6X-MlP%3h3Pp`g<REmgTS zW&kgqpfnT%+gvSWP<@)e-=3q?FDRVyN*i1HZlz8<FjQ6zSn*-_<u?5(*9khW=1ld} z9Rw#gnQ4zFxOIEb*RGY6Pe%#M)uV^&4DbAa33F9{KCW}-1|@fXs)=cfSWV?$%1#3z zE3mx~K3aUrv$qr98Tq#hGjq{tIlzX^dzqGIQJylsV++d2iz|3l6ce-`Hc@e203aUn zcm_j_Dc9{ZwnGgp5f^LFT^4N%S`D<KUQE13G#7$hjX>Nv+llxIcTc|}gP1s{g0V8A zz>3OP^h8r-e!SO#hbWdJNP~oyooSK9HxL(p%!O?l88=WR$<ftieA`IhQ>yWw0b-|4 zW9+|sz{>_O|Ne6|;*@tF<?Pl0`u0LoF~n4pcAD-&ucmNYY96_LZ=K5wR!dM2`&aki z5}NCW?G86<`yeG$OWEg>VtMu$F7mB8WyKux@8o#(uA$3K)BEB~JAUgs`kDbx#P!@- z7$P?>F#XOLUNjKqn)VHOGcWo&LV3W+L`1+W$tKnr8{x5AU2(lnw!SI(aKaib`6}F3 z!d05E!0GeFNH<K$atPj<POAhX_$59aGt+297`x&xH)nrWJ}!T09y!O%YF>)CIs;N# zx|(f?ic_%t2i##biz?L~B5I6a)8Qe4?-EcwE$?ggvi_3_!#Br~U@OOiFgHW1`&w-` zP0V+(qEM15u3lq#^%%x$sK1~tlmkA^-Zx%;3p^r9nG$-WYNx50GpRuTC=J>Z2gJTd zZ0_1zTbT}hRBcoq%k9_}9v~W}HwsWW>FQwRKC$svbVvrWB&;MI2Kn}o(c;?vk`a0w z$?_)&^zyxfU9c557!_Pcm`U{)$=jPUqpBug&s71<pRNVa3=_ar2D+$Jpn>=<pvyv< z-Fo8F5LEpef2(oDDdP9lM(9iB`8sKn?jznw`aYqC=oRm-rk)#O7Bfe<ny_2HR0o~l zT-=rHZ`wiD*LM@4(YRK1f`iJR5~myKtFEn|SHG`0!Oj2MaQvel{(%teO!UnE)fNBH zh5zD+|DUd)Q#NsT{>NAd*;(5;D%l$tnf!}c2stqjaQsU={ACt@mBe4{@h^SBL`x6F z@E_G7%0Np`!0?ag_=lh{{!{qJizxnO8UIUWF#K=Z@gJJO@E^A#`xlJR3H)PMZ2!U) z0y<$6R|_K(MKQs@@%|sA#V1it97O?Vw6~E&unN?KRPk1}<?mkpl|F9t)|1_MOO;g# zM8kB?8g6!W_V%PL37%j$t-vxvhx(iZXDB#4E)YCoprH=YDLE!cU~2po8&l_X_<9iA z4O8z=?~WJ04!KQ+r$3;_AlleBc_$Zd|2dm{7#N5ERkgY)aRP>Gy0wGxl~2fFR_5Np z(~k~pAQBLgivLK}=n78GvwPp(M~elr9B0+xJf>k%n@mw6!@k;(StD=GvqPvB)E(oU z%rFWHSv(OQIfVw<F<ia1F>}yiYD6bvN+KEQPM?w)nr`|t!mJM{`%^n<W}W|$TVL^V z2K6G0HV&{Vr&YsN5f^^B#g-x$mv#b5G>{A5_|cYX-ZyC7PnY)hM~C*dpAbh6FuqGM zx#Vw5iFx$Ci$R^3ugzUSr8Fl!NCkP8><>}TuJwzRD6uD!N@pZ5CI>b2I^h?3S1=4u zn66JqZ6A2;R{&xx|3w!Ny!;NE@4hKysx<hJl&Q5Up%gClRSr&#U|k5adcke#b?rI< zAqzVr3mvnX@?ntHHu5`Zu6xh=^&5_L0o&-<S&0Xzvd#1AI_4yam)WhnZ1-P-UY^-* z0$pX6K0(tVv)@ILJqmHf4;egZ_w2Jar+0f$gXBYYUJUg{>vP5|Jt|md)#GOKtw_!- z@t-5s27l@;h<0YRX<3fzlQSlU_fd6+*`4jp=})cv2_t-k5knyZfRyLWjq?+dJ6Qu) z2QKSVIFWcwi*`tJZB|b3A`h?Jw*;7DLt26$q`ygjQd$p5-l{*X>WeWdEYOCMORD%j zgdDObm)cNmib?5u1U-HgEhUgxeCLr(sFl}pE~#6Wi#R&olHKv>rEaEDL|d@US7nwT zo>RYzdauc5;H}k6)v~TQ+GOjdZ^G0};b*AqiK~6)RSy2JaP{*2fKSx{n*ZOK`>&+< zCr#PtS^t~4|D@nQRevp%gpiP+fs=_b!M_qpkwE9aZT>-6|IP0IF<MUlqptpKnppp) zEgQ#Qb@eyZUH+!?U+^SlVeIsmS^Ya{|2M~C_}`rB-!X*@oDHn)%>E6)oc<%f{s(X| z{O@<nf3y5QJnY}azwZ+cM)v=AVrP3odZQe@{di4XPF{AeZ4q}$xUywN;aIO;k_O{& z^&J~RLI!|9(5~?p5C%e;35jEKvkP;8M6qc~!8b2ML=w~!$qQFkn|)L+`!%a*Rhsiv zH`CN*{xCK6e>}Y5H-1dtO>^CRY|gx->jWbTayEe~5Z9`2@rb7+{qWD!Cbt+Kc)o1r z{_zuq*r{ov0<YElatN^j6l_ANDb;JXT=>0G{dG$qSOuRy+3wT5#e=`~(r*#W9oPSG z<TIO~NU}8eBxawz(CYa$d%;c<E#CrJ6IYF;KKCUws#B-?%+aRX(&1$u-#DiP;Q_t7 z?j!SZzP+L(*3#!v=*YufXUpRi<%LsY!1LJM-`pL9_ErR6^Am~Sf&EUe_^QCZKPPd7 z+2q`C5)}hbJ3f-<FzarfF>?cTuKPKTjKq9(*fn0F_5x<C97pP6CyMrz7TD9SkpR1< zKq)D7_9q~V9N46Q`Vi=xn;hFi0d8ExgB%|<kD?3#HQ!Kq!0b--`et*&I#}dPYa9#N zFWsGtA^B8-I=_`@We9}RA5VC6PUG<+VhlE>Q0d&^7bnbSFa2FHtWK1z2p?E?PXO;* z*Nv+)0#EQr52J1Ex)%hGzi(`89@(75=|+HUz&3t<u8+kgz<TU+UgxaOsY({dnp=jt zLt-F&d~8?r%HCF=9RpM$)hy>DPZR7ZXV{6L9_(YY;Eg78@Uy^;gR|{EHg@zmzrtc% zjX>N$xY=Tl;HgfyXCC@H>1egU;Hl^%@FTJ_&?y$!xUGPj?}YaX68v4>(<x8dmB*)# zBia{C&f2&ZDy$P-kSIbFrC0nj7X*nX={IOgPnNgjHCbz(m^fbs)dl7FgQE@wT6e5b z(JGIq(?yrd_H|G8bqDW@tQ|WVEk$e-O${^`PYwymu<a>a=cxUp4x4IkJ)MnNHlzNe zBD1LDMe;Hq=?+Q|vfb*zU0-%G-8D`oyT?PSEl;F=bx%mtZ<60=7fx{uZROWlvrG~J zj<s}PUUnD|ugl-H@#;6#aaWrfH!dqT(N~weto0`CD>nDn-pojA9O~#!arw)`Mi}q8 zU1J=oeh-NZN>%VP0MjT$=No8YS$Is7NE*8_lQ-IzH5-_jS#Awx9iAZ&s#`EwF$4YY zAaWRr88^eNy2@?~l02qy9O$kXqB^Ss`BCGDHhaqW1E+Vw@GSR{bS&ejL})B$>^e&P z&~0d&;@ItIGU(UyFuQ<L<ak6vpJgKj{pxI!b@CNM0EJOkb(y?8ddE?!VhZHU)hJ^1 zY;kEaDUpnsfL88?jhc#$_LM9=FQ@5DaXoB)er25V%33~VP6a2RpRBY2GaGc`GIDXY zvY#{gVMsT&YoFFSi7j6dc#gWoHaXgZO1#6OHWVK0amU4nYgl#CN6BDXR5mgZbL!)y zO$jjb{mC(^nV?KtV8Jwmp_mIDvpvcSW_rc5!pz(HG&SU{Ai%_al+*AjYcz-4@hjeH zDmDvQ8MEJS{Iun=T^aVh1WP=f9?;Y7rwFvG-0#jwkAed{5<9j?1L*`Gd08iYA#LFx z3MA!1F-5pEggm<_y+xdY=HS92qBy|_dQFpI(z)+GP?$u|CGqn6Up{=J>31x~ag1g- zuK49n>n6BoeL$3VdkCbf=sa<PCjz;;PMF`PNpkwpNG^6+?qOGZyaD3pB1}i~j@sq0 zrxnyH76Db9n?<k}L@DB$>{4b{ND-wGlA`dOX;zaiM@Xe%y*%I`$RS`%R8CZZcWtOg z9-C;|Q9%LR2(fH`scGGmB{)mU+r(N<RKyQ4@7D_`wc%DT^3PLA>4`g=3(ZDWQnu~P zo%Oi}t01p946*IdFL$!-5H>PVIv(DiDK0xdFWrYO4;m-lZJ!q^=;P)osq<HGf^@C6 zUZKSvneEN<$aV-ZkH+lvNsePm+YVlH2$RqTN*%-+1wAwKG3zl)N0&MBC`YMbI48At zMl<F@sz$2H+~?cw<D-^<DuIS=?sJSOjHz)E(WH9eTa#oglV5hsmM29B`x!x?=!o2| zogexw3>&{?q)8paMjz96&{bSdXnvEUyo8|yDG(!N2ck+Tgyf40)@L&EHEN*dlk35E z9wk~FRZ7WS_Z*BA*FH(Yo`7!;R+Z8FnH-+GsmaewWXI8X+)vFttxPoP-c7q-ls|tp zT83q)emmxm|77?5ybwy>mx(+_sS|OuA&UJmY>2QVC)Q<4gFj85-Xlkh4wPDzCKm~& zFWmQ?1XtFi^huE%ZaO%Y9Ik1}oVMdD*F=8?;ko>{GwCIv)5R)8U*n_2YP%Fe>Fm$0 zvkGjh8j7J_3+93tb+ZoyOC%n^sH*1F<*`7;pq_?_VfF#S{_Su)j34-f_=i1Q3#`eC zV19?THd%x_tabrGAsM6rm$ZHWSD|2|AGR82H176-IT%cmL<1|ij98L;co63pq!8`_ zE5VWT&)RQuwDVx@CS3vemDAmnAykJWZKjp!C&x?<*p)(-ZhTM`z=LwAjk$Aq6B75Z zeT5B~yVJ2GvLtZ8VE+O<x+nQ~;mGiIliKU3xJ$2CV*>+X2L=bgoy^l?-5zG)?dX$0 z_7$0~(IqE*0oKcu6MJ>KdsQyiNL8(x>P^;-4rO&}8dho+8k+Dq+Y5p<bDXA<)vH+l z5EQgyQ1`LeWfbhF`ubIsODZ^2coZkw%$i#9JBB<pM6+}`3#j7uiaYm4r~3zSdi`?p z8H$GuC7W%#HZ2iMHNE<xdQ-X4uLwW7xaxDQ?U1LiP~J7O&Cj@OD?Gb0E8AQhzXH3N zkfl%+i;!}P>EbON_LoJ5tgQu~qv#U&TFvD}9FLnm9zV;pg*xwcODBKmHS$v#VL*;3 zXrzLEfdESoVXdY}mnZWWO=<Q@)8EGL)Yoy8carsV_OQ7`rhiCT;njn~Vn$+&y}TFx zF14IUXfvSE1!*wkfeGt|G0|jDmPSp^C`G*2=jqWvuFH-|a?DGpT}bEpoU;w*ch|ma zZSqy4Xv0#|oYqq;KYM=GFO-gLE|l7t@wI(DN9mk8t+Nem;y6y23Thw_5zk|==o`AD z)`qXcAfmyjrNdq>53=f7a!GxUcYE)}CCxbH_CIhoN$AcK?wuKPs?oy6W8m%@&XX6# zE7;1<w?yG8mZm(<J~;!cmk5`Ir$vCOr_mUHMVm<%qEakF6cmB_R+<0NXKsV31{*In zS3M&etl&U3lyE7-IxNAcEO==q??4k|e=Q;6NAzl&nI=uKlYYb-1%pV@vWF6*+Y(jN z4G%KNDPfU}R&(bCV9o>(arwi-RK=2xXhqz(R5!b7wO9lmKT<@96_bV*Uq+6>LkrKa zu~~*PW{v>x=0AE0=^@@RLRsS|N6GB@-5{708?<DvT7-0@MlDwu>U%0LU5C~P?9x0{ zyXp2!-Q%*66mvDFm#UU*tLu79CIs%?<87*a$8Y20gyXEMsQI@0eB8`IYxcQEq-W7v z(P*=)2J%^fvPEGBRNo%ILZPms6dc%uqZV7MNX{lB2iY+WeFEW9Qj*NMlJ#l7XE=Xt z?be*Co$XuPxZ+}>Fxk4sdy3^;OQ)^ZFtv|r5>zfI@A~JGTtB^QGzt3;aiTGR*dxj+ zjw$v%yKvLC-6Q;a#3~<bLFk!}lS%Pe*G<=rlOA)WCk16jvj2X(KPZNMwzTgpELa~K z=BueL+^gw4!`?)~n@n{r{;PnYVWj~qZQJ!~)D{&{95vtH&F}9Mp0pd5CNkcxqir+8 z%)_{K-%q8_z5?*d!;$lfAL>rWlMd7g9M!Pq+YtPm*P@PByBHm|N0ARE7|@peL2?a^ z`naXG47ySSv_lHi=mIHgL<-HwYHJH<VHv&0h1X8Y#g|H2$jBi5ABWGv#=E#(i<uBU z$DI128Ertk2C3#Z<okXwPt+~pay%<Yqu>LOY*y6L1!h_u@fF#8rC^G6N=1n;Y69gb zlX)1M;9w2)G^*(oqIK6ziFFOKR`Et*>r_t%zB7|A?cUW&cQ`Gw=y0oI#4F`&RMjPQ z4><DCctha<NKMLf_W>caN<l#0z=2Qt^}Yvlnl_m~vnCiU^xu@-XGZdg1HX|r3N0C< z2Gt-B<7bkSRY@uJrfP=Ve$w1o?=IMou=V<Wv)|&>^r7w^@;<K*8(tqpg<Q6xH?HO? zM=ye7P$a<$u&9TI9))m5cr1`)rqVD>&U6&G7Zav%$Tw(UwZQ!ah$-Rn@=BJWtA?zd zAt^9f2?=P9jTTIxkkBL2Sr*e|aLMH&tspT1YV;^YO4;kxfJ=M*9nuaz##PFI!5S{~ zQpHfj+}pdxvop|O<AnQa-T}3VWrFq8*esQWebXp)zdeY$yZhKST{oyIy=sRv!X%-f z!K{vPu2Lv<&$nK~aWY&A4uWn!bDnlI98S58v7{<4HgyVLl9Ss)q&*BYD=7G0ER1A( z?9O{sBBbNvSV)>Xd6u`bs#fEIw4Z@`WC0&p^lD(@^XkV{&*73djfO}&(KTusqM<UG zoWT<0vQ11mvq&BiRqHtCbo*rTtqtuX;kMx7kd%`19CeShA}A>bli+@*)+zu559EfB zG9+_A62G)e+PdG4*KE0T)Ze8H0-Nn{xeR`{<g&KZSap0TC^|fBIB98064YuQ)J|!~ zyHnhKx9>1Z$JvdIxg%Ct9S!N7@~ye$E$b1_Fnels6+xI`LtSxLFxk)258_E%4bUjA z&Kl;Xw}7-_tM2O#((9UuGo)7$;y$?J5K>~z(b`ogZDzope^>^_R$yup;(hJ}_atW~ zc?QI*D~SHIUk233%&@UG?<e|=#<EG_%29XRydH#6vb~ouuNS=|`3V^v;6EEd=ji7A z++U9DbU%LHG|pS|GX3#I{p0&wbot9sqq&gTsH1b3fvRw>JMHEGT&4gNnJJZZ0ialQ znOc@ntN;3$<mlQNN{eR<2RKnIH3W`>&cb-x+7OPMnbJ9PqU=p7xpD2_VYG(xW$&}& z_FyZ@O?YJV1das<XTu@yX*TrTgWZ3s^Jk^2y}JGFbqgB#aNT+oKgaKx4Ys2DQLbyo z&>Eyz0Pj5ch+DOY&P3(?ylRcQAkNt3T<hxVUHv@Xmwm#&Qis+^G$zKPIU`<S@oXN~ zJWgy(h7jJxpw#5zNZGmP(vewqAd39iJHRI|_^~5FW6pn=^^A(z^{jpUzAq=X*)N^c zC0i%^RcjxI_We=r#}nv=&wc|D*Fo21v}u06c$f|?S;)Zx_-#h=^CUN<b=&V1j0G<= zU%Ta2qfbnj7P0F0GAxe1HCh8<b-pUSlP6$AgSc>^Ie*}?0rRh*DmE1iTWK~Q)x(#M z17Cnm-leGt<->iclx|9obau$tu71W_M-jk9qIS=F*6I;xB(V`Ov6&o*Cp5>2q2x-t zX)bpQQlEGI>&Ds(kYQ3Jsa5OvT6D%Emiw-Z<aI&6yXg5k42g6t_DXie4ivF=SuZ}_ z;inC;Hi1@-%tm)>P)D={^Cre6xonFC439}KM{2mC(tID^RK7zpk?E%D1^!uug-$); zvWHZYnhcp5NRPhNAQOr9m3^#F7N&~V4oniH2)R`5SyCCE737bcmCPmI6$EvauSv4n z)1DqkZcfK)@0C65Yx9pe^Zl(Mv$JwCn*MU7`&vjn%$yh=^~RT@c37G>(Bf2Isg6b_ zJx(rP%A(U_cJcz~&w*IVLu<>%4c65%3A#y11F+~Ppw~Vci<%~NECvhQcieIbY(=P5 z%O!ASczQoP)v459aa$uDBQzFg6GT&dHhB1Y+&%kudhrRzR~{is$ErH-phIcsnd{r` z44CNof&(i~b`{ME<yj$v!prAW5J!Ysv61SVVTO|dMuqVqZV)9i1veu*Do(0p;?*gE zCd%b=wSHGDTd(Y?uJ3}hiyHJdL*+wAhDs+gVlwy25fi(APLejP7C5Sx$*mq~%Gq6% z2asadaLGxF;Yto3ePq=~Z^C|iD~NdMJLUBgE$JkKR(HKnokLU7XRP`MJMq!9^{9`Z zQd($Rw_fn|@hOrgPG>Ql#hNogK|w7~ma^eU?+2UlKgaoBHlKFhe*)d}y~-p+u9ms& zN$=06>gV<-Q(2)~swEm!buIV7VNiz0A<`r}0qqgX2IXfhBx0br0}La~cRn@bKrHre zl_{x!?&8cF7DGh|S?2h|_}w4g_Rg<LkgQO#Y+%Kqld*(N(0d*7L3+4*XSGd_{zUPq z=e9+(>N+CmeJUm%BiDVv%}VD-vLHu;ico#T8=;)f;KBr>`SHCii}~2yXWuzsa9NmT zDBm^{Ez+^g<8YFYkeNuX;!b-N$U8BkpL8%X1utrVu2ClqmcR(m_SYoNBefSb8`lL3 z!EM@%FPUQClBNzO%v~{#mC%q{D$-*vGDH0Vu?upk4461?!`J6Gd9*5lOyB-wk!mL4 zp|PmVSaj&7X*Tkm=tmn*ES++rDEask+5B=`i`)CY`pdY;K}$fvK}yC_MwMG!^w^q) zoKkTzT|_Kn-SDdZnJ?#Kv($gZ?vCR6XQtHp2c5SV36Qq<Fve{aOI<HYQK4E}+XNyF zXqnP$2!S=E8nII!CSnq;AQ8@Aq=RAWH=ZQExEUH*FUwWsO=&Wl+n>zrNpNyVQqsY> zj3GTf?%Y>^(wrZJ(rnM$7A`#Z+h<_-gz^YEBr|&!*1=Lz2hO{(8$F+=m)g&7H;K=r zQv<I6S`u)!2?4(Hrnjv|i6onpPT;zkm*51LGsfpVfRAv^tskc6H*HP5QQikEQ`|F+ zrb-KMZpVV3H#ga<O{O1tkh~clqax2yu7~5v87grUr=KkZOSw<dySayxbE<<4C!AVB z7P-eV4?`p_<vA>Vc&1G!iy?(vhS6$$@Ypns4472`@Ku3N*UQ_pm3rXkj2}4xKkvC2 z&m5j!0`803$mvWh!tojGil4n8wre}C<Ws^}ab))@gBz`clI6h9c`3OhLc0;I(&?^s zf7%Zm9q9Vmz>*2PA~pf%JWc?E@TrvOEQpXUl@>V72-yP!w<7IWba59#@<ru-&)z*+ z=W69u#hz1Ne#Rhpnsild=@&^ur>WxJy3;t$Us<H>&n(C{b2|dC@}bWYR_;WcHpE-$ zWN62)!{=EV`<q!N+KWAX3wv6p_`9lWM%Q?HwL6@XF&QhJY^YUq>k)3Jt(-0Bn~^up zmI;J5P!k1O4GylZy44vzR^_MZ`_fG831l)su5xJLI{kA5rE6%f&+AMX)Vzh3Z(qf) zxG;x)x}LhrLno(9Dl~jo<*x6YubIDk%TrUj=9nCvbhj%4t%5$-)TYzEZ<l!JLW@WA zYTGQ`6!seWnnhOFn0GbYvdZs!I6I4*jZ|G-Gtx}|^j>9j>?!IEGv7G%m!x>O=+^3_ zlA5qg0ctGCe#Q*72mOh_Ix@sP*^OkcV6m`#(u{E8R03pgHI6k!0`A~Yn$5&X%v0D+ zFck}hisM%+nfXlOsHl<_5lhn79FsH|m%`*1*VE@lg7<Y>7YY*-8@+>#KqQ>n-ehbZ zh5Oxgv<h8xMUa#3{;nTLtE-?Ke}RIk!wYq1jJTSBHPhtXVX7_!X@<<fv};UVHj)El zt{GOx^Ta5x%G2b@m;*`%N%;|o;Do8gy#4g>+sJH02F^@OYhf_qE<ya_Mwh5|)XscI zSy`LDHF7%e$twB9td2LXWsK=z6{ROj<^HKE;-u96ch=<8i6PD?mp-wRydbnZZ+0x@ zDb6B4+pYl-L9BINnxim5T7eNV*%9@ea=s8|89wZ<P*+G%K&1>puQRamQx~kKkdx9m zqQ*pap*iF2Xi_O76q%m$kb<-r(u!QkVBZifhVYYpc#X$-*A5DcA6y8)bcG@zuEQnI z42dWQ!gN6=4ZPG}C>FqNk<bSatq^pkQf(sZOU4`*i0{LAXO>r_pE?aG)#$8I7exj( zDY6FA2mbqb5hZjg&wa58Zg;#n1zTXqi;uuyk*^m|kqq7n3x4HAz1p*-{IlYF6UcU7 z+;_73Gw@{F><>h;-Oy&)itc%HQc(CW^R~q#9_3(fc7VD~r?79P3Kv7FmP_oz-Wj#! zqAKkCz|?)<%qoU4%;+NaX`vcv@K!{Ijf)W*7Gx_mXDA!Z!0e)Gjto=&?#0Vi#NrSM ziuoab2HeK-2*TkcsLHJ?=)*WMM(OvO0{=7`v<Mia^>Z9Y-~nG~ju5N-#>|t%3*Rhe zJ4d|)0VtE2aUY=Xlr8mVKhm*p`eO%XP)0Izm?>~n{dGjAxPw-9ub}Iu-*Uu#SYx(5 z6i1>tb3m#66()@A%U<uF96v8i_U-Tqt3t~XkB-v0n3s8!9u0qM(DOU)E{K~>5jIs; zsoir^7`;%$9T_!3bne%QelknGrziqR^e@hN@V*H7ab?p6vMa|}%XFM|P!_}z>X4;+ z-4;M%$83F1wz2qe_zswJ(wK+C!>G*UB9D<3z86Kq=h|DM&9MS>=2z4dzcZL>RdeuQ z=^vP;s4}4(17y?LJo>O@NVVAzL1~QQpunQJSxtIIqD8|1$YgWo=<v>FS{w0=V%*tS zOAg1NA`ne-$7DTWN}n+VW{n(~eu1MC1nlQoK0Pc_w&B`Rwc(XSFS~+%uNEE7RPP(G z>YLe7zAHx6A83vwKTUQU+o@h}WrQwMbmiD_Kg>UeKM1>+0-5GQ;%eLuCquaFUpudM zL&|_1f28{MzD^y?KgPf<qnuowe>LIrcP4GeakUd3?!x7{T*5iW(h9LPJe}hFKs(2F zM^5*5+guv)SL5eVt=1~wcLv8c5;Y;kR<2?$!as!lcA5^3MdtGi=4Nt;j43qR{R}?F z;jDRVWK-yU$B8cdI)7N1^;U*{!3(Kp`J5^Pj$In(fF|bz4Biu6$<xd0H4HX-wMi7P z9E&Rtn$8z`bbW_k9{C>8X!G$#;~V%9P^xKHdgz%Y<0k)9okVU8cauUZm*FJWnRPX{ zTS7e#b&)%-em4NNJ7xG}r{H2nr2*HN4r&vNhkToLK0;@>XC-(q5~WVMKTH73_rP_5 z&4v}h?Y8pjs6Fec3W{hqhw#~8MWyJCwjw;oOu)QNgyoPY9T)I0hoTH^mIQ=PE`(wR z4pG3Z5Q8m4WC>^%o7G-~VQ$V-WyWvL2Q??O48!Pvpb?dlC)13qkuNF?y&hEhJSrKy z9=zXM9t;X=#p$`ZUENEC>-ACR^mx4qi(oXqy<2Q@RKvjL(#9<+?yk&+p&ojn^)@C< z?<M&CcmJyq)P|L{v^-{s?f-Q47GQBL%i1smcXtT{2u@&dcPF^JySuvtCwOpo*Wm6J zT!OoM2%dk)KH1rOpL6cL-#-s)t?ugTp6;HS;d!dww_2$x_${wR;2e6H;Vdz4Da>ru zQcTDgqEU@nHVX?{3Op_PHKUIj5;Yv8Dt%Q9gumKxe&<)e4ygU=(CBEc#ILH}mYXLe z(VtpvA60=d4&0H~4oo$Xq74L~HP7DLA2>Gsxqvi&N?v^{L)KF<y?|24kPjE9yo98e z+&9<uch)Vnn(ee0<A`}2BaK$??Qm@9t(i|*P8m*}R@RqJmbSjO8671OcvvsCIGx?J zaYkG^<rFV}o~utBv=Gv2vSW76>^U{D+dn2Z34KVqd<UZ*HEBO>|Hfr){(Xd`UFhs+ zSb0;~p!yJOIi+RTY`^9l&92>qv9$%W$GF@8{y5~WqatI7(%nIQY(7X6W@*5uYy|ac z%XIdZWm5`AV!|UCQO#xrlq5&MCTh0b77CUJ)n?8B3Eib(Z&0*|=6>jhqw8XU`JqBM zEtw&LEyDS@0qzR&D0j_5>C8!9uS=Y#TZN7<1U{F6r~7CFjs+LATS7eTJ8TSu2n1H_ z9PaTr(f8|)AkvvU_o=yPq7dR`x*Xga*x7-;jg9uV*cl7W7Am_7eqvuYt7$(;M!`_j z*j`{0_Ov8Lz<h*=D=ZRQoGaI9tk~CRU{Gi6pa13*eX)AZvD2NUiQTHlhQmm;Ct~v% zI$G`MzQ4@&CQ^J(Yc*`{{i+mQ-szWM8x4hQBktqyXV&QJwEh1Og8mgtW@Tms{E9C# z{DPFc06_i-2EX7zgp3>v?9FT)ZGd@4_#F&h(t$A0(=!9Z%X+qAMrJ0ajznziOfQ&@ zKN@KP4D_%s8A?omX+i+NI52RGKQe9*(K0d+(E{jMiP)K#h*(*eU+^d|&C+^SKv0OZ zvaqPSD7CVYy`i499_%}DU?AGefZy80(g+y&ey8AQWTpI4c(3QGVrB?Tio(DQ00y^J z|1}8~R$$x9Xh1}X5rF7tpjsaGFOlotF?L2~U^_ki3#jFPx-l^`5xrc$wHR6csKNd! z5V0__{L%hr;kWkB(%<EO8xHm#s(<LS{p$@`*xCN_&djXLf2lD67yf5wSY~EcV4fAm z-x4Ag`oGnHL$Uz(<If&M>}>3RslCYm<6M4sV4-IqVgWD`z3lkUCp!z#%Pzi@Sr{3K z0IW<zj7&_h%<O=lWp>s-<Sf7s`=<_2j|sr~vp*xyT3-6R=&}Kg?&bOO0ssx-Ph<RR z5}e=02sDFVd0iCkos9lyc!5j2eDMD?p+C&wxBbHYv~VDX>8JTP5Hb7#d;&s{>>V5h zP4(=5+N+e_zbXs>p!Et{1C#8SSp)G9@AVAiZLIXHe@XsgT7Va9)Guhpzd#Ydjrxg~ z`UQ$$0MG-~{#UqDxw@Mtx>APEnUa#qH?5U}hq6;?m%?fB=D6|q<i(`v$k3v`ATYRM zzC>Arl(1pGP`a(y$ViUpCg%cL9H5`pho}%aHW80Xw&?fS!XiqI`#*sii%5NcF5$>J zZO)u{u6~j~tN6Zp_TWA5K?#IK;MT#&iD_1q*+9_sMe~o@xoT@wIY*xB3JM|!xN-{A zSxkmeCw^@CQ4LSLi$f7o@!5POr?E7XEh6SpD^L>(>-%@w$4i+X#ljSI2&<Kv@EEw! zht?FY{p_DNjVvl!Ht~!>h6}S>cT0BQ?weOnyfCtDvEt)K?khC9oT@v5F1ckfD^4}$ zcrUVBe4us$rXHvhE42mR$~j~|g2unosV?ym-bszqqlMUkrquT1h#++*4MX3LVS-Fy zR=zUeeWf`4!LPkcb9Y^?J6GWF_4Rbtj}e)Nj)wgh!iyN(7?d4iG=Sjhn?_Y2HiFox zb4!Sh&l%y36sLWULGEe5D#ZgO7s<KjM>BtC(~;JAuguF%JX9(u@dDydf%8*HN8T^B zpk9Ggy34PwDJw!Q`5j+95xy62_#(%I!W}t~3(5pBn`QJFac9d#mdu<VDK}{&(#e$l z+De$0pU3s+?zmJ|B{V~%XP+x;Q^ctVPNWHpJBB;#CTn7{={R@so$RX>SnkykZ-4zx zVHc1BMD#$0q#TUT$z?;<!r$oEp7!0YQ5p=7{BG>yg(w{%)FTqJODAc++@QK6Jwbwf zE5h#6%TQ**TZwYcd630@LiM=#c3W5nXklwXcXmLGXoWukF_+Hb&FRnu^M-gm(p&>o ztCq~^TOt>?{EU20>5l#+l><%?V2c~Mj^oOLku_YqWT|%Z>Uti`{EfM2QV-t97X-ec zr+ueu%)qstGKkNp6TQ^uZ@GM<Kfa}kHvUApm}<xa(?xakMN##Kx$1#)9UQ$UDmUB) z^Yv@rR#Yf4l0~6$c}R)8LI#UkyxND_cP2gvoZTS2Go588Rn!(=oJ{eW5$0OJZoesA zgC5V$o1)hy4=Ahl@IMki#}erip<th=%-EazF3FJG6W<FKWGtq*aXD~!2s^9Xlppi@ zy1(+o4@|H0mql=3$}%&BJQRTgJ)$;xb_F>R4cMpG{Z7ac$|ICZ5x~u7f??h5aPI!- z;EDcvz=u5Wycf<vWKj7MI*??W%`Mtq*1E)(Dt}l0yL_~)%oOLRpizlkLSxwrvKHD} zqFVY|76-xIK1+|A_G89Hk}BNSWg<({EKHz9>A@U&P*~aD0CQld@4DO(v-XJa{gv8$ zwZX?6UOj=PLu5Q)+R6^IEr5iwRdk7KA{|-wQ*wbeZWVt<yD?!o?e=n|T=li-ZCno_ zdPL`ib;J*}#qRK3>n1<9Axi7l;lJh7gO;=;<r5GhJwY}W+$VtphOGl>NL&P5(~ilZ zhUkNe+)EM;17ZrIWY6R}vb|-V;+(%&>>nNY<VR1FA535C99+Rq^nUs@N)z;6E?2d5 zp_Hvtdx&QwLrBo~q66gp+7DX3qrtXqJVz27B>Nl4qAZ<{EDNCsw;-!<V%(6P8i*U{ z;3xu=;243aAUy&M_^`WBk33yRb}ea$Eo0^ksh1Iu+LhDhGgM>)RM;tRurcbv)jfh7 zi&)=>yt1<DlV|?|h2gbUy8j+dcJ{u9$1Gdr^zCC+Av8DEQ)7mHMWJgk`EC%Y1SS>? zEhzDmGZhj17W;HFk&g-07ur10?MFlQ4y|zu;<KwRIDv30I|;EZ?QF7@X6Fk&NgLQ; zP8x}l&!2%gXkug~!3yOZ)`I(3#PSGf%|wZ2V;i94?eQ=y4>Xcbljmk*?e9HT`h?Av zTZ!m-BVWBY^Kv|PI^YIvmOk@u3J*^dI}&ZQz=XtH<)O*BYm=`yyR2ZEolk$;HiaK- zx`Zp1Uw*_}>!s3~Y|*sk5ov~Md2;mH6vEWnW(30{Uw(~1);yEAvub<pw1TO<p0g3! z?S!4`9*kE0{h2&*IXn{tCKAmF#OUO81#!r}Da>_#^K?<J#US0%j)2{y-q*-2Pmtk_ z&HhahN`k%Gm1M~ut_~H?=_KY&Ym?E$R+-)~STtkk$si+AaB$n=ud3$EDGw_}z~D-< z@gAEQWSdCKEl&y7EjG%T(6+(4JK4eUFtG0qLzuk`n8S#C5XIHZIN#ZU*G2Dy$bByI z!lvYGhGb|j^}sf<ArPVbu2)4r)7tb!lc>5U7QGcANDM_0{hmlgm9(zAL?hn|4KmOV z1`aAvS&zYzjUZmZbpQ2!6ev4mb0XjJ;*Mna%DCj>^XaKNHaS(K;K&xdV-K|CfQTwp zGzeIPKw(afvoFQWBy%}+8|98vc(JuF`0bvUsLDNb*1`%xTiCmD^eF^MO9Kl!KV~c} zfwzfxMvxJpf{sY=+Y+EU_k*3j4t%iznr_&+KeUOXl;BC+Ky(xIT8jm(m(b7I^TT}K ztp^x673!9-Rn7OEBs}51c7^@!SIOsxXw4&b4lcqFVG>FwREtKDr@s$_E{rb$!3j#j zS@<|L&#{6TDb1dm#+8!Rohc)|D;ifs%$~A6A)tsWgY6ki&&6YWavo%pd}EV=SK$5K zO7`Y6$GMgv*JPKbf02L_x-<QtG-HqJ574!-+B`j(D-&7EOEEL936mSA^$wGGp+<4< z?}|4LkFX0G%&?(uQrYU|_38wg#MKscm3I{_lG@YTla#bi#f(M#xPFLA@hWeJ`7S75 zElkhghch$xTggwMhck~DOf5!>MX13a6=PX_$SJ^u=?cv0LdyH#Zs|0!C9vHigZPYI zP$dah2ik-nZdq2m$k9aF?B~<Q6otm-3rqxu)A&K{aZDXY#lSfb*U-Jo-{TA7<PZ2~ z`KwQ%?gVYoJc5dpCA#KyGo;eh13mjUs&$5`cFDF-;Ik|P^m<SbHpjP%D9?}3=GbN+ z>M8S~vxDU@i~a@3SdKxrL<CdnN*K_e#377VVhpZh6uvrPK=1MAIZ{#48~IYe^0MPT zSoKqT$jY=&$`ZZ><;|y0ElUiP10d5}-1QKH=_q|7!mGHMJ^MPpeieQo!+JBX6L)v} zE%GZ}3lAQ6t@?dqrmog*xX1JTK?6MF>AT8ELP?Vh&U0Anp6xq17?pFuVj@N?uKArD zuX3~S&_Jn#w>dcJ6JE!K&+M5`Y`XWVoDr3T!>Km7{j6b-<-u>5OpMu-4FbH#X6U1M z3nxual)r}Ga&-P+SPtmFMZ~tYOpQYju^8KSAX(dVm^3xR2i3!@?J}%R@xu7_xm%VH z_7=lZn4W)k>wsD$x-!+2+IDnsV1!wUHMf=C%%1q<l7eba7HME)x%CLb?8K?+rZG8a z-2f>(&R@h6qZ#+zP%dVy$G5$2d+(WYRgy>VvqGcn-^LJF)|Sqwmgi7xXIVN2tQ08g ziRj5|78{Hfv>~G0rQw>VZU{fX1%l0EcbztCRc{sEU76^)5I7B3A;zQzqkzTuBlP1M zL<0F-+iQk6s~63!-|15J{L3(%)rzNzaJZHH5^xvPQ$nD{zmAaKZTFRHsr7BZhB_*D z1s_$@yaIn1rYbFquSC>|#PCR5V`0RMOnka)D13g38^wH7K8u$hxU~79+sC>dB(rk0 z-d&<J%m}smwT;!`jf0nX<J+rDvBaeQW2_kwV-hNXA8&&j+3X-p00u&Dl>4_?Kh5Ii zuO6;{yPp*)oZbk;AM;i3H_9O|T0U|_4OyvHVS0e)dOr?TKbr6?=rL->$RIfwp9M{% zAWuUw?VxP@n737Azyy<Q`C+krQ?v?vR9Q<r9`utSWM^?fe+&pa)6C$91b@d$G{Fkb zFf>o?%z?d|Nm!QNChV9%&kyK4Mi9>I2Oywbpo50h6w9qR-xbwSjS?_Ck@?_24VByg z>Sc&fo@IUJ)$Ub`fv<@0V<p`7Y!?qr06N^Z;lg7mVQIY+u%KAHLF23@EThYuW`ZVP zdq<CaK4ofR(?0j@HlB9Plt4-h72`a9?91gWGrVaz4w@c7qPwNP;?HR8QAq1)0J#pz z3t=`**l*UZPk3OLq}7|Z1v{pdNKvrI&D)M&uk$lR4_6kK^8*v5tVYyJSqx{Aog#t} zCy+<o$&#ifrX2Z@SL{#*aXthK2Wl@L1dn^z#k~N3KeCr}mmk*rj&TRGm71~G#@HZh z(!ylPYpqq}H5%51$5N`{igFS+|A`4^r=pyr>M@%Erva}Cr%9oPR_*dUN2QaMleN>y z?wQl6{mE8l^dpshEHBPvf~|Avm^P7Yvi{6X*85@GuHq|4RaSnJts5bn)WpSL_yr=f zZvAf!uf)+Xv38jBkZgr|K2#f9l?|}G3hX>I@&<VmBLs2W%BxtkUQf&{CQd9=M%bW< z2B}pPW?OZ~MSdi3u-K~h23JYhFO~v|44UIhtZgSD8Zr(@c8!F2Ni1w9GR96nxY=`P znDTP9({ppecl<l|*_y~oS`W?o&pW|4`FMAoy+J#ZUIQYCan0h0v%CIi$%MGe7f4j` zZr1@1j2*4d6^T_%ox3yyYd<EWMeEhBlE_zk-r@`l)cEu_%1!vcug!s-6ozG=lQmWP z$`6(T>vSj(Jh^CfGw-=Z*HLsYa(gC~W{53o?N^gdvc~G#Scl$s)EAqR7=qWQT2@~i zHpP+`CV`8mzgCc{-!~RXCR|<x30sDyPZ-cfDh9%wivC^BJ*-i|d8vr?Qo2&%gS;F^ zPEoe`KCbXoD!OrrvHDroX;!11Mm(#9&D=^^$ES`SkC3X=rT1qur!yxrXWK3wE^v30 zRuZDBQ3^z*eg2_ikkKnqX5`-j{ap_0rM3Kqq0sgr!qXQh3}-iv=nK)=f^r6-2@<H3 zdZBHoj^iPsXm~_IPB0Vv8(@+J^9IfF`Wb~I<*zB(CB7U{(0wT)rONB6B#HRyiyubu zk=J=n?=?Zacx()~nivP^1E_kwvDC&ejiX#CpLrakoH)n&#M^uZCVzw2+OYhz_{CvH zyr{R0m*R;s0y$tEP>S!A4d9&%VOv3w2y6LwvaxRtstgUwjipYo@7~nv9?ob#I)aXq z9xjRe;3-@-Ok=-y3u*i^8QdczIxHF;{mfjXc@4vCzp=!uRH7SGi+Mzj(f(}Qn7f=O z2d-Dkr}G&(Cxl-|b&{@H`vP)@^x?u=fUqkX8GtlF(i`3ui2atMzRw%cQr3>>3?V#D z^$K3hk9|B#X*<F9bk3aZfnE;bqcc|oyWD_1O71-U#Q6uk4nG#YS!DA@LP#R3>ROYD z8xVVw3GFGW;51x6UPr1>(e_sdhkmblFB7&qcOWEe(S0WHCHvWA`PP6OdmWxS^UdJN zLK{?-la^JD{yLMJQ!U$#D+`PGuB?&+W!y0zsm-yQW1XU}MFB*w-<bE}+D&3*!x47} zYhVbWEtD^5UBw?#ev4beO6ZVp$MB%`mT;r;uDPb(5!2*NrKX~-Yi@n}PGbL?DsOgK zUQI!rSfv0gnu!GXV(h&Hvx-ikwIQBGcPq!7Dn(*NcrWaCtk`Zne<7t>+GDA^)ctri zsdHV2xV+%;D2aYbH%Ms0_Zd;*xZ<$EX*{t?({??{)(J{epYMbD{QASxP-lYCi4CBS z_L23E%#Z+k8r*~WV{jLQ_;guJ6oCv=2cICv1K{=IL~D*}lm^bl`nmU7trDHE&T;qO zE;@j<B<LvNlu;m!V7_L;VY5%F^H;hI+16ItO=E$Jreu~{Qfai23Ep~t1lT7}+Tx1) zF)Sn=N@ePdHIV7GIh#mS7^%w5PpqnyT9W%#Nt?xMF47f@5<bDPvX$mAMb*Jb=Q8Y5 zIBW{Rcyd8-X@lv&P9u-4!3gcAo%+P8aW%q*kmkdbz+oqDLsFNJVgE>vP_+O{iVvsg za1&-#0s$@-cD_C2V+4aijZo<Pn9{VcwJ8Ga?#xG;y0RYf&3XbtN^IE5y;7p13-Bgg zUAEB9;I|jo!ta{+?V<WRMQdazZ2ZQ^OkMf6VDzw%KB^qS={2t62-K{h69h0KVKafM z`t2q$Q^XR*K8li(&~U~YgS8wYOXBY&=`>ym?N$~hcD$R>-{HF>(6DKBcx1VM?Hy7i z8^MDb-SA^8>CAZbfGWqUJ|F8oY;CdL!qM3wc$@wMDCEhE_qHhLbv)aK^O5#x(S}#g zLxT*w372J1voN=0n_`n<eQ#@TjdqiC{c7uK4&8bgi*lcaRj>Jofs(pNs_)q9r;saL ztwRtqSXV2MNk5{@ONnLN(FvUQ29dpaB;2-KoQO3XZz~L(Usu^Rdr^^ei?F^H;CL-W z6+r$j@Y<h+v?ZZ+)K-_0tYe9^`aOI|cQ!+S8l>`+`=n5gkZ+k!1u0<>NTt@fQpNp2 zEhm-?b+OYdQ<=whc4hTu<QwxJyF=|MuHvoT`aCxaUOI){j0p4f*6crCcLbv)T86{c zWGXlmdr=_OrQ$s)AFFlK&eE`I8?|R>4V2GY(pxfF(_7K|lyX+ik8fkzwFT#1Gi<e7 z<Lr>TS41h4l(9$@zmdSm2MIxb48Mr)-ces4H*lfwE+7gMDPOW0iFwQV!vaHD{DWKZ z+6Ah<_C#{3UKu+lAb4>wSt?q4o9Y%1tU`8AD1v<-`cp&=I_Jl+fD;xG(V&>Vb$_nl z4|CLpEOAd)Y4Fi*gqXDx0c6?WsaWS|Lp?G0NSWZ)7@vNe?w6M&*{IjF(IMaUOz3z% z$W(I0p*!B%)z>;YG1Q;cgJpJjICU6rK`l7lzE9)L?y$e_o~nR}MS>x^992gbm=9A@ zqM|QwRs!#YmO~iCxTjc~Ux)QZ&+?R_n9yfDt|6OQ7@2WhE)zh3Eun~(_3b*y5zZe! z39n`VIj0}bv6Y&87?}7rb6&G6GYpl8A}GD;%Z2AN1Br=zn<p`Mg%NsL5}xs)myt|W z{=Ib?J(T@!maf^=w2WxskBXz4&ud!yEc*h_;?hq+K66%tPg8w$G~}$zysgJ;8UuI3 zV8V#CEp1tLd2ZMJ2?!DgyO|;hpWYrS#bk5M1RG~B?guWgW*3*=-4uLH`Oc10;9D%u z!kJ=`k|eL9#B1%oTwpo6CAk!qgSdox$tiA%x~5tX2*CK1pQfygNnDF+wK_O6NR2`T z@HW?P5Ui@rYZqbla2^DhE!}*{K!y<);h!EOr3yNYZBDErv%&5lIgAZ4$#a!lu34!W z5YQlPn_0o&<XK{TH45%Cutt!esp2cXSsc9OyS${pZ9x_l78&&^G7aqwFLUf-j4sn| zoirOdUDs4*-a@3oDM4LC&#rG@N&xHe#3Cp^)iitC8lS+W90Yis{V^#849&M)$)nf2 zlWNa#>y8~p?eEdTLmo2MSc$pExEN)w%H)h(&7#{Txhw1QaJNRzM!4*4&prxy#1-_a zf2q0Dq+e{b_olZw(WJX#wR1A56_%tIu8J~4rWSW~Ib%MnlbwcbtKTWrmK*;L>z=@Q zv)=P=m4mp0j9J=%$ra9j1@3*uZT;2rjKWgGv(gmeqdB<u6Z|nhJ^?B%E#{*PwX!;a z^5*(&GOP3}1iCZ&cA}^&$wxe_O9BMCXygwppY;X!ykbvfn`XX}_?a@H+G0}(vq@hW zbHmQ3F1n5$$?ZVtJ40E1%?9w`M^47qaT?_l?bS?*4bLmDqekH%9({)o@idwWFq8`s z%<jXi*$xmjfm81xq`@<cF{tXIE^T)ik&VWyVm9F``*DS-U)IgWy-}f@S}&Y>(w3JE zbRAFh!{o;S6ikws$3NBNdKw$IN^+l?ABs6a2w&FokM~5KT3?Yj)@#M7wJIt$X{Icd zUR1OnDPsfNs7shD(%yqMk#ls)Avo8I!b33KLq<|lY(s%}y*(#TPO=J-Jnwc}#@Kd7 z+13xZE2%o(fj-odTA=nEJR%cE>dWKgTGv?Y2_Omf#2^;z&IE%{#%1*;pTmfk(qt2D z>opGzH{|_96E-(193GomQKo#jW$vKn=o21JqU41GrJ$xm-l$DqL{-uCnq`TkZoIVA z&O+6?&QghXZ#X@~Y(0{u7F8+NrMJHPPODq}3iE1f_uXa3kck;FX`p9<Ktko$8G;T0 zRa~FWvdg<CW|25vT^5MP9G~Dx6>~sR|CpVEnHksmcRv;M%WYMWXfEj_Nk1wQ#X%)4 z4CPOf+3EIBo1~h%rwBH%uVEB6mZ(C%&lF6l)@Sj!fyZa}^3^lPezm)2y7hT%;TGMw z3aBrj!;G_rh|8Tb1l#YgC0WC@ua}*$ZzVbO&3!d6f~+kcx3eaQ$j1!6vnBAk+hk6U zyJ0s9S*c0FG6o5mr@4_%J8cgz+LTlNK}mMum;q~kQ&h$BVcss}Y>9_<8lV2N;_zrC zY6c8WW;ksszZV0K(1(%Tx&xe@#tyb4bS7nl9sBB&fCL?+N&^Bqe`XG+49K<2rc#K} zkK``oQeF~Ia1vSe*KpNePm_;{z8@0K`{}xSd46I=xnjOe)(sdTzb!W2mVmyiP){NP z?_5oS0gdFFFVRXlxsTn<qK+<D2$XwW+!*}LGG~;$Xn65bmR?z!Q>)XzqlhV3QXgGd za$Q+jY-(s^ld@?cHiR~GU#*HcI6RF4hJ0FZFOX4F*8zP&K-ZQ)&tsc&;}{2RjN!D* zveO8Y^;stnjmD+|5^7-H^s=Aw%JC!AZq3Zjjsl0mtjcTwfRTJAthRslj=C>Vij!wY zURe{dbpyl@ECo<0(d3XQi<@z+Uhw&SFI`fJHHx?U-0&-lzWvY!`OR0Pl!_ToU<+kl zM7dKxE?hnccPM%nkbb{JQoT8s(HJkrh67A0A4w^ZPt4di1g2N@gBLcWDmXKPRSXjs z^5m-+)Nhxw5ul<CBW4_z`BuK^kqaza%7Q!+9F+JZxR4d7NeIVIP!m)Y6#PL2M{qFD zcfZ9$Pc7*iTPHEUF&fOR97HU9uv`N_sjL!@`*Bk8^Gzhlr>oB`&KtQdm8y*%qA|-M zSAaEdTN(#c&ea~IB%Paw?9mG!k4-hr#CXS8`G&~=F)&R>#p4P^<@?imUgTvgPa01S zPrA`!e<@e}7<aDMjoSvi1FRNWp>cO)V-+>guUNW>sA&sD%5#!Y+eu?7Zg82Xf?d-z zy?mW&4Ecr9DW(~UV?3FP&Oy*{qRx|(7K(BnA4I{#dsw%z_hajYAM+&*4nD-_@Kq~< zqIFyH2stJP#=IE}I$wa{Z&uP;jQjBv2lAW^qN7h24o|dj!VrAelJl(uick$;=e0ZA z6sjwbZjv^DZ<Jh6!+r%B^>o*C_(65Cj>|xG`Z1NHCxniAm_jsDIq$8k=v!f!ovY1w z`RaP{qUXz7yia>hyN9D2&!}`U)6o(jZpfZl&`WQnIS#v&ypJPw#c@dlyuE8nL8S8d zSbf{Sc0IHaQl(^`6;g8O+wZ}oF00xOtn{{xjD?h2=JC}J`4LRE+Io*m5iTPIV)aLT zA1t^ujRxP`o*ocYzKPIkct#dlj%yKq%x1<9FgkvvUW095Ts3^Gpq&AqSmN2Aa#Uu7 zK|`iZ{kC^X*R+PLswkH{Zf{G1^V|Z?7*IE%STtH&eA~7HcO>y<wwyZdQ?uGwZ&FWZ z1_ANbzUT%Gvj&YKZq>AmR9^p<$_MjIb1scam2j#|4xQ`Un9N=&=~Pn+fS-o#WBeAv zKyh;}i}&@u-QeY!U?8r@eEU&V+lDC{oeoDp0c6f7sQK|DjSiHgN@p-1Okx@}k=&aY zifFX*a^{Ziv#R*YVoeQGt$un54apS=TEP=*JZ=Ex@+OixS9!gp`TdZlL|4&FLCVGV z>7^^IA2Qd3`#}^`0y1F=ON-8D_|8!-I4z?sQC=}#UrMjd+c-9N73=AMfzH|H9Dz=7 zW$p>1C2N;OL2K=N!yJx05N^3M1+40dDyo8aRIr!sY)e^t639^yg0Y%njYqM5P$yLE zSy?7Kc<{2-v1fN+AL;L}VBQCr@-77M@xag%l9tYyp0a&#NP?~EgIv@fYANZ-5_`(# zpvi<G%OZtpc=O@t(jQJfvP;$;(V8w@T}gXyUtg@meYbTnuJD%D>ULtLCpT68qj=xV zn^<1<@mrBTMSuH)Z*TK;id`Mm+0=9z)2j%i!H%Y~!gb7?-v20f%HVNB#A9oEOvxtk zq|9f0KWZ>4az|YruH(dC!iRHkRXWeu_Q_-c>v`b2w)HOdrTCs>4U!XIbHWS}E<Oqg zS7x^Ylne<MggoIAVZbxty~1|RsnCh+fB<elA;@ady`sdK059dc$gO}vd_uB1>g-jz zddam8{(T|V91~mORS}wPik&R&0u-;XPdP=3@U`W~IMz`$+IOMmvFiCeMU+!%Q{%IQ zlVcw#ZF3XWX;<LlQF{d`f`U9NTp<ptr0VS=q=T5|d)3eirKN>AlW2}~w79(PYT8Rt zhEOt3M4x!?-=}Esq+YE(yd4Y1zl7m_t(1*!&R>41!ilzX7GD62%yFdn8Fx38Szrg{ z=4js?Ntd1VB9Oni{U}YtVtxUkDz6y@!M?kLE&iOSSfAea7321(zgQ?|V<OQ17W!lR zf@DBaO7JpuU;y(;$g=k!$~E*yLdr&z!9}sWctIDzXX`<mH}HA*`jhrWYKY7u*~aWM zUrY((Idp^FE<Y$_la6sH(e-7|faHJH4C}IFpy=C<O|lhCM#~(?+!%7bcW^%|rFL4y zvg6E%*d=*KN3aB#eE@jl!-$2SC2pk(rDr{feaS$tbAelm^uICk<UI5ZF5SisZ$><U zB3Rl*v=R(yXb^mOMF^GAhN+NK%=aMg=lq%PLCEhW_|?hMD=RzDR&MARiwiV+&u9v% znP;z@o5;|m2Xb?p^dc=@xfau$IaWE&wfr#`vvfM(5dar4-XVm`I%I}O*7K(eh)#Qm z@%^olJ-0R!WUZ4q5^T0BEHi50Lt}#JG9Gwiu8d({uql=T-v~+A@KLgBg>2co6GV-i z%p>-_IbS-=z7HW*V^{RSUH^=lFc>Krf?Al};9lfV(I^*+W?+z*jZU85C-{vsyUDg_ z-^wNzG(-*tQNW2uVsH#JKBnbiK)z4b1iG0KN-ukHkiK7Vb(F<60@3YVZ-K`|68yBR zLxo{-z&6rHz02#E?>*S0IkOKf3z}y(N969&-O!az88_{yEyE9<A0N9Fim~!1C*#<n zT*A%I464&t+p52|@|Gs0eka|&I6aAv(?TiY+HOM#R?`ARm>{^RkmcdbtWxXs<_n*! zLddph%y>wKm9#SIzjH0@Q+6xdHBO0%jcZuA0<im_)kj*}7>u5jm->9=q=SfuceBWw z4{6<L8XA!>yp%X1cgy7?^cvJyP&@qA6R%2hGv4Z-J}_=rx&X^E9=t$wq_<KQ%4pA3 zxW$wfw#5b^>d@JkX2$kVjh!&#FWw~?>Ypg1VV^-RFJ<2xs~ZJkA-=Yfcaff0kI(>V z(3V}m{EbBJOcnpBI+N4O<{_fun#aYZS!&pDW#!}m_JE|Y=QjGbHgQJJQ$(UU`?$x) z@)O<sgZ#=$!_9SGRE3^rfzc2>6U~8SxcnAQkyyCbmuzlsNkS;$EGj2#CkdDcK`J>R zTqxBB`BP_&3JrJ>KWj?LkIm^K3{4slX^Aauv`%=-m}cS#B5&-m3c#)6S}YAD^5{TZ zt#|BPnVGS}huRd)@RQ~VRPuNHmM{52!5xcb#IztsP`~KW%e`eCXt$snW|o;?mPtx| z_fXM9I^Rk@-wKtI`$MMVx!{LJ2UrsK=R4wq(%Mx%NnWnDgRk>+4f(8NSb<$*NKsx< zn1|^dy4CSF-wI#DUl&fLf>G+7*MxZI-4Pex)$o}kbI(jIGT+85Wa2hUwMsp!o^&R1 zsx&<6R(z!!tG^t7jdLt*R+TWNsiE$j*CcGd`etnQ=U=_>@|y{=$+;Rsk?F(NcjZmH zDNUs+6T&|rPN8vT{~g=*6L|9i#$l%i0@40;?i3Kp^BbY_8&32a8}ui(jg=XQ()kT- zV_{+aE41yU@h7wm_8-u;7hoF@UIa|(Ld3$x_IF^LgtCOpdm(B?Gb<wp+MgIAV3wEP z;5Jy`nEwIX#tK}>pWrrTAWrQs;I^NDqnG?kf3ELW@+A!-0C0}1!1eyWT#PReE?_?Z zD-eza1cSZ2-OKfBj9=xyK3{Zywf~1F?0=B`Px}9<?dKRQz+_YZuVXO*@r5sQ1wMa{ z@lt<T<BRO&`Zf1o>v<XfSKBYyi~frosQYt2enZp#iuU?#OaBWr?LQg83ux`P3Bvv~ z(%<kjAY|^3j9dQ&p2h&Q%wJ%%{~4b4>%6xYVCOG*+8<}D{i_ND^}Qe(|5tQczABuj z;!XnZUQ2Y7ONNUvU?@=B-XxjpQy>hC8>nOjDOv;y4Dvoa@~0T^SC`n@Jf;jNZ@PTd zC)G=w@}Cam51x+^RAIuBom-xo8XDBgn^$h;oAc|NH*+9PDr1H+Jji7|>sFpl(%JVO z#_uxrJb#>duC@7s=!Sr<^Bt`&B={#)2Qu-#b+3NdJ?S#aW+nvT61v_<nOTa|-#I%4 zTSo+eo~*~S*61l&U;sHl_qD34Y%JX?`=0(9#L%knrrs;>yu23c<KqM=){WJ&_mR1F z&BsT;(7esU^2*LT1az?5OEJ2Y1t*KJ+HlyRF;Xa2C#z$vOxH8+SPwBe39Hoxg3N7L zv4V?@rJ1?a8Rp>z$XK0R_Ru^*`eOzf)Q!4HD4NU}&()rFpYOIpk0y1mKDs7Y-j$x@ z4=Dmup$f62kobb7#c^O!7$3fU6CJK5Lc4(>`|hgrbvs_S2**X@kdG64=IN);=ikRZ z-KOnL8+{U&Joh<5(9ttO`!NA^*4M`7OW+D;-AnKpk@>hE{DPa*DwuqX|8<B9{fDM~ zp<ept$yZp_&{vb;T&m)4M?2kWcS+vzM%S)KAJb0PJdpxVi|E2(gEMnPm6q;Nzb6Q? z;ltll^(9;Io$r2_Y4IBg(FK>5Psb`6^XWX>z?b-pZ_}^m>C3+0*^qX4R}FKf54RxP zj!feUceSp5io+HC=DrF!Bm3zZPCD}zj4*i37UP(08Fs#>lnssf)Xo~WutsHJ3q)Pe zOUr6(^sodcVe$ZD=cAcXK3Sh|&_;1~mU>JkR&wmo#L$hA8@F}ej^l!rvRciDR}&0@ z_A^iKFFtBYk|M(fp<={c#>)(IsN{KKPjy{|O8Chz4?o1G{h(}G5AzH_RCAa7T)epN z3a;*58K5#iLl7bNrNvhy#S;BU?R8Z7v_q<Utho6kzOujjNYnnd;%K|5<y?WSLS=mp zzDOC%@DhcnlLE;C0Ba21B6JK_kBFUW*hxmnM3^C9?r7<m)Tn!G1}(w$hdh9T!Fm1g zO_p(z!>SabQ?+<w@)qR$fqD|+LjcK+K@XG&hH84mf=4&E7WI&W7SjOAdYHZ!LGWt> zNU{fi4w>{+8om_WvUhCwEXycIlBh_m3Z)aXaq8A#NJ0ItnnR|NPows)iu)haO<pbA zj&{(Qe(32D<bOKPxt+}?4j@N`;eGGQhFO4ET)S}ZW*Ek+E|*blHU-K(^sPVns3pR! ze-0|-`}Y(FZr$;>fuwpo4pl`rr@)?hRELm5ElVv-iz&dT+l}b6MkxRai(%mAs{v zg|SH_O-0L&sQrpLkQxyh)RF@B$*3boT-sa=r9~5)$@q%|!G70i7Gn68UWvL*)v7!8 zX$5N+ngmh3`435IEeiSWt_`>>1AymEn~Zt$TW0-__Mj1#lAp=gxt3888{o-9T13bp zl|up~TwU|Zt=KRmr;1MO%$YeZMxT9G?TNCfhj`M2+gXvn)Ud5$LpPx%_1%oTLC%9# zWtqmmeD$P3j&=!f>=op<pP6!f!{;4_cZWD^A`d18zU;2PCEkd`b6DQ7nNcugNnx1f z@S_N_RQwARW*?(HvNBYZA7<nichbn%>JhT`e37BfomlFzI6d4Q%B#HV1Nn=D4#}aL z{(Iwt<R{iJ55}G5M#^yr)i-elpyPvTBz4n!`|K#gNg}~!DB6fe5<*y;27c_JS(^pW zx@D`ewqJ90E_5w#VPOF&;*tpX$TK}!UOjOdp?orXtSJK(rfFX=2RVoVgz#bQ*QH5P zEV+Pf#HR2M#IyHTz93kU=#b!7ihK*bLe>N-QYTkT5No=yq>&n+;jnNJb2#Nu&H~rR z5fXWXpPjZ|wG0Zu%Omine}QnM)7X!1S4e#~q8z$GKsu}(wMGlApRQ_ShL%c8-HSLP zVpqT8Y!}|w41x*WqKg%D{Ggx=jK}Q;roRbG(}m#IrRTV?7&A~h3`AzIT|_x9AqX-p zY-wi14gI2t%gN1-Fe#lPPwtFJxn*Er@HVM`I0nv+7=AHXHD<NJ`EjiNK&uQpGc=9O z592^3NQKVIaB=G_cFcm@o7xg}HN<?u1N9ABzyN_nx(!oYvnY9cC2hiq(_~X`o~Ms@ zGWG6AUJ?EnQV1GNmrqy9poa42fUlG-IjbUP#x@;E-`xwBP4)sX5-^6s6d`ov<UC)y zwqfKl%hpyID<NPllVv0qr}2I*_Edz3G#M&=OG5GnB|<2$IEbvT%($SiqM@ppin9>J zVR_>~6+qCVARLb7r50S#T+(1oH>cE+F2-R!%bUM;da}~m-Uv6F%j&IUv81Y;(OB1D zo#iyy)l{p2HGUTYrJ-?}!kkMYtKL!2?nLC{q=F^Q&F$Y+zk=s!ZNo(Ur1Yazx2o0M zxu>`dN_uuqxDaD^;;OJCL!;uhW=O`-XM1827Y=!@276#l)KpSn_%nj~t@mR^k*DKa zI=|Rf#4vy~FX+|*%f}z=IQe|IFDT|fFun*0=X72j6EN0&Y6OP)w#KrbPq8<P?X^aq z25nButI0RSTkIdVy7kFcToQFr6<otjQ9i0MsTJ44cTTh51qBzH<#z^0)R2o!K;E9I zn*lzKf4H1<?OpkzOBCcQfLLIgy+|{-U_BX27M7TO@G)D9S4@}O(RxumcsDOJRQ^8W z!t0>tmZU&GR#HbFJlV!U5&>^`A9Bf9zfCjM|Iv$yU)AraJItHVO*;`?E$+dnV)}&G zFaw8I&>&jH=qBO<ZE0%qnva7@YDbigZd-Kpsn&9n_YUomu=$&EH?CFt$h-KxdySyE z*1(v-7ID?RX3DeOoN}XLSSatTl8?uzV;d#ZlrXPDDWXUg7Za$I5k)eK!<4g26={v~ z^Ao$4O-y2tOu3<rwv?2Z!fTnLO?#6Qn{>>j)-^#Dq9}t4G3Q6i$ku0PM`>8rF`dz% z16re59yjN5cj^(GeWK<xTin#2Yc^BL$3oVYXTApHM-#deUFf?NfnFe%iMZVB(YUKz zv#-q01di0!wC?mQI&eRyz2>>@tDH8D=^MX`w03M7&hW5HV_oCiH2gv4{qB|siN#D3 zR;^ckY*#gP;gz0W{g7K*ji#!e7OpD!Dk*twEc@3*t4rs-n=+b5qIVM+9=*A~d18`z z)@G@sVyHfY8MGsd0k$PX>b`Dl`|d1jwgTiQO^5qC1TKrf^n=;}7I`*kYY2hfa6oEg zhvNan!M1{iPNqil)fo2y`E-aaWnwl|?G%#Tk%E^8+2dIkSa|M%&IHAPM6<7n0nL-n zfuv-k<%%ubqEm9jNUjSO3E{*Sfo@)7KIt>#(V#~XdJ3*rMMOQ$?39JjOi3uyroBuC zNV6!0$szqGwFOWYlAho798L7U$#rk&AAW^=XG(y$8v{%fqtx^&*PdjqR^$%SGLF+O zp%sR8`02Ccs)dKYSI-R>r?JOJt*RYB{|0isvv^5w-jQKE^$NSwC6BUME_YxAUb?FS zoNPwzQW>WEcRlnZErI&-p@!auv3?D+7BdLv{PAH+W!P7+i+D~zf{W;I7pb)@n;C1H zwa}3??DWSWE3rw?;P~nfOqeMBBpSEX(s^05k$U7-eBn)!^nu-*jwbgyDtL)oL_8_0 zzGus^?V_X2b+8lpSAZMF=!37GV?AOFYzGp=lzHGi2_|N(5m=0bdBbtuSLZFnc!%}$ z_pR;hr}5FEeZ*dUrEFS=_<<&m3D0?%lnwj?Luv5#JOB<xJ_8|?Lwn;3LoSxxqf|-c z;TxEY>i}t&Isb)-)y00-p^peM9lca3A^yS&0UMGgx2Nyw>pN~rLci<4^aZ-$wKWDa zMW4l#1tQulR&d*vP*jO}HpKLkRG}*3MQ*VKh{tVdmNfxxl)kD^Nb@E=lRahhU`zMu zx4tES3xNyh+F>-Iqk>KNBB_w0C*nUw<m!!tcfd^pwO26E?}O5GQgGUqZ=`TlkZ`Qg zVqa92M<3rcUlQ7Qo4#Eru~1H_G^zaKaUZdD%vnQMbRj27`cVXq{lF9vHTC4Jlr-he zRg*J^UE~*hSPX(86&gr^R*}30wrJbM>?v-AWe))b8r}W!np(sxDy1OIExu@x!FrQi z6UMKM`T*E$b_7S7DY<AhPg8Td(3TWL!Rgfqw0#}MT6&?7JUf9{Nx*X!ZwKR(z<mJJ zgbo($Xh*?whVaq+=mAAr>9YlrtY^*Y^9{54gXB@Wt)!KVR)_Nh&#~L(-ZzEn84U(9 za`bd<<?TeTiYapIAG>(7cL$74)x(!lW<04fl<x75sGnSuT9jPjU)iQBTVuR><*+Ev zVyu9lNWxM|jXX%Kk)C45J?gTPyDbY@K4l!PXY9l*tlP6NZcguSXYg$?81icU-Eg3` z%jbe6aK%BdotVm9>-74E{>--PPOv&O%>0fX&GEZ2H)<{j<)nvwndB`l+HY5T(;6lN zJsW_ISxwGnc=e3+mM+ABI$8kQoo3=#Qnh|ND4rS_uSP{f!<+OD&(vi@t**pN>K(E< zgtCw|A{RB6YV;0C;OQ9#4Q<YKNOC@{y&5$%$6RXFD(Jn!uDM^rkAi3&k3%{0tXN<X zVl8XKr5qWCS{rw!YQl;2L8M9cDaW%E9NwnqB&(f3GqD&e&$oVdvSzTc?yXiR(7=Q6 zmF>LM$PnXATzzV3oNAd8wcT4_x8%CQU-P;^an@VzchR|LB*!&e(2O}nu^%*sTP|mo z=ynoFlZ3Wl^F#NiIA80JL&p!s9$04>nZ`86)!(A=&!pabBf#CLefuWYBdtoSj-kiD z>KH&{JIjD<-kYqK+6jdnmwPF&*ji3s(?<`98yQ}Ng0p{Ptj=f*Fq0Zi4j_IPys!oR z+39iQAbnuU>NDDzape!33kG1~@i+X}V@Hxv74~CIAbz7R1wM9-Nt)tM2ZSSN=$$dA z(n%fDa(F|D+ERoSnQS}^+qZKj?I&(z^3@900_TvHs1{xCTA?k1G}&mMk)`3jW=nnO z0b~<wD(+R;lHliG7ulxmJAb&E=%G5b`=F{F0-Fs9J9-wvpA3ey;b=Kva4Xpm+t$gm zX}ybRUwP5@$l!-%jpc1N-VG;dnqk$6W|GAZCVW&FNFjU_QJ*o?6sCyAvD-MV(@arQ z=3*f;Ur-U|mdbjq`HkCNAFB1fDRZfHw^0lNHXA#K9_%m+ry}TR15y<_T$h0+;9aM^ zs@o=#4OWTGkJWNqy}~`G7WvXA&UU*m<?TY(205=$+QhDHgw{yhH#Y>n9SI?E<12RD z*cRWPclq9T5ofw@%U0aa3Qo^JAz>8jWVbp)Bgr|<hun5{X?}YSK>5}lK#YNy!|-J| zfF*y<NrD@LyH7Y%n5{U?utF~IO#fWw$wz;9pewpQy6G9NC=DsYr0e_FHIvEjxN91> zgerGeGZTv!2Kk58Th`3(trxEHG6%);fotiMiawI)+y?Nrj}L|)7{2r>#NVm?aQudm zlk^a9DD$v#h&_2F^gSbkZA^1wqt}7$lAtl=9+7Gd);s*9qg0-tow{UoR;T<F@48e_ z$OB4amJ@2_Fr!QDkjSa_Zhs^9e$qkI?F6r=48CGU+1@9Q(B=EXEa!ct0u)7Cy2!Hd z%xdPb%r}QH>OA9zf?`aDyxsf4VlFKCwhS|Kw;&7kUN!YyDmpt%%%+0i^sbmvJ}1b_ zOM>TRN<<y!sGhAc`_Jf&E2p+gM$|#5;f8yzU5JCnVHTZzOm=*RxU8KFSWHu|zJs0R zDA|D1`Y^&w^vLMF%ImZgcN&XTtl*c=)|UTTD)+sPUt@r4B|M()Q=y9Jk_ee<nTqg| zW0RM#C-_AMxS3N!bYh|4-uFm=1lRi!m0nSXd5ZSwM=IeaIsDI8r!8PA_Mt+gI%j4Y zsPhVUFm1zwp6tA!DDDkbYYkQ!+m&1-H4N?(n{Fg$=l!gubWiEFSA#8Q$!nF@E<Iyx zi`R!d6q4!M9|1o~2|paufBVYIm^KN$?gR2MmGVENp$4)o|8R){IjO(>q<>BC{M)zn z+e!7OpA^Uq{r|-Q272>=6kq@mJ2TLI$U^^jPpP7)xTuocFA6ZNfQ_Z$|Ahz4$_jLa z{TCiE;AaZ@zmhe7ai2ASjMe`W5BN6?_8%oCw*QSc`;QL)t@_{Dx2&xH;_b2m7xLe_ z|7`fn7%Ysef9SF?{j2bwda|(nbM=22;y-oxhm3{guLR|vjVvro{~Y7bJ!WBL_`Tt* zz$E1?Z2vYnW_BP=n3?4-W3aRSi>b`Q%<va>@C&*27vc7|nf@;>*#BT%e>l~Dn>Q@r z<qPl^AMA^<0IUC%A`JA=|KEJDe<j!RJ2{#H>GTwujI1oci&Yayz!rG98NJ->Uv7?n zmh@gq)-N~1mmAT`P3YzJQ`z#R^i##)rS!9}CL=S$OXcSnKik|XVc!Yb8|l4}_$h#E zF*3Gsr38jKUPNX#)<Q3bGEfL{0O;vi>Dd{W85rmRj11KD0CIYIa-iOO8^aeB*#AKV zc-(}clYtR%cpy)jNJfE36=;EUOmx6wBjlX)>>b^SWQ9Zof&OmbF%-bbD4IC}S^X6J zz;h@}jffP@EcA$IiT+Zf{7Y=Z6*ygBj0C_A3tZE`UPM2iMAk&VWDG!)|EG-p#i9Q4 z`imZb31~Y1kTJXv!T%v+1O{^cA!B1@1A65DuE)*_B)b1Y#>UF>Z!#bs{-5&$u+g&t ze}exu9wXxm5&rKo2DX=<%fHL$+1TlU2KBeT^z5v_h`>K(Y|Q_*4tDl`TN?u_GcaZU z-}?e(O#j>$2H-F4Kj*^0#t6(!@b|vJbO6BLt$&lT|8sr-;Bf!^JOFF}mVdqnBY^GS z_Jt9^{&GUX%j?h2mywZy2^gLEhl~}N8RAcwqrDz5KZHH(ODsge%-slRS48jRZEPHY z=_LO6u8Uh6+Yr5M$IBOATnKn@0vD4ZtC2B)-I$(1j|srSs>fiYZ=}bn&&<eX0AK|e k>GQz;?<&CmbijC>gCp?E@YA9hnSh@^EE$=wj0o)i2kir$CIA2c literal 0 HcmV?d00001 diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/test_recursive_embedded.docx b/tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/test_recursive_embedded.docx new file mode 100644 index 0000000000000000000000000000000000000000..cd562cbb82d3aef15b41531d3cac49a398f82fc6 GIT binary patch literal 27082 zcmeFZQ;;a(nk?G3ZQC|h+gNSewr$%sR`+V#wr$%sZtppFPt1+kaU$;9IWfB)>ZR(T zsxtG-`u@tV<fVW?PyoOHAOHXW2mvB)<DVM<0RTE60RWHzAb>Q5Y;By3ZJczK-0h4V zwdvfftq2N0fGF|+fPT*Z_x3;d2@Ir6%kDEE3|))A!^Kxu4A4>&24y?}v=4(qx&bWb zG;BTcuL3vNPPdp-2d$OJ6Vb&dq`!7;M->{0CI+((G7l7WR;<=w)c|EZNVcm_;^BWK zrB;elWFZZ4m#rcpWk&gCMdPIaDhMpnEn3B3RB^~Tz^N3Cf!m>&do=)^vQ4=m+;yh` z^5etMwLHM}>W8ZbMvJk}gpmghUW4-5NR+JA?zK&htFR&yV$gyrr15h<P8B`K6p~WV z=h7rAZS`C};ENl1mGZ3|Wp_2D+x|Wpxypbb^M<k5bQVdj$YNP0{45IMh2yN-*3)*C z8b)ml*pn6hpr$dtMaM_oX7g1m@pMzC6{aU7^?<J|Z$hMQSiiYG6mMaP>&huuyles& zraA%E`@0c-12q4>MX?p;8o;tuc)v1iZdJ@p*i)W^!8Vo+;fMgWcXJM@u}l0p-95>V zuR{OX+$Z+-Jo6p@b(!YlegENk!|(BRKwbj&MxZ9;Do9`rKBFV&i>%5NN<(>1_ZDZi z9+qq7d4E2&NXm*sP{8+5yvmjEF#0L51%*<EyXDpB8Lj&x>F;k~0Qvuq*zw}Any-Ht zEc>Hn=pV7`Iv86y($oE8|8J514_5Ynx%J9~UW<MPm|xd{pMkTTzt?&(3T5dHXVx)R z!67syq><NGESA4Mc~(|{v`_WL#^>e}XFMFUM4fk%v~RFdlwl(}A(vit`ZeD>+yIh; zJBk_ImhARmGLN22KF5e9D8~XL)lnm+vB2VA!qO&tkPoSZpAJi*&50<d7Ys@23v=w1 zm!H#onDf#Z7ndx=H9f)q@QQSW;I>U+`aqfR*QPSl#>Mt+)oBkXoh7y~MO#ol(qLIJ z(357x8&pDJ-E_>Jc@%b+4u%KQK;}b+)4zL}Xd!8Q9Y-8t^0QqNsyFEF+G1yE3*7%p zeSB8%6<YlqQTp*l8~_MFS6c@o`u`;oBU?jf>mOhIhavt`Fn~X<_;dFE?5#m%T(+M9 zp@aAqKK^<@0{Sq&q?4d1Coa#+57`z;$V%+#?qu6gqtQ9Pk%MCxcbvH}J3RUHL&{|o z9D}-nN<7De1W?(WPDwE?^{{^sR)U6UP`wlt#RxQI!t~g5p8N!(O{xjCQwArp4%8g+ zom<j+F~Epzk(a_%!IBuWZI-3MbP?OqFQ(2)FnU_R1Uzdr0DXX_WnQ<+;|0FC%JO$_ zXVn%d1rL#crXEg|p-~B-Q`R}B94=!1@Z-I`G;rh63HHjwew=^VTdq7GaDmkFm5iI- z$cVEvaa-W<9V*qvI)FORH)W1%MrpA;c_Ak3cC;SL@5jqrttRoBoZqxNe}K1pJm5y< z491{ObwWDZK<HM$CKTfyM^S}0dvEETP&E3KrM=~~I7R4#X^|=gl;BUUJOnC-VKIED z>URSnuv()2ep83zqA=NWe-xrv8REG^L#0+;bZ}kd{e||pp=Wpp`&ZQ*9A~|V9s~ej z0~!DT@rQ)}!PfuE(;}61n|(F}Z~7Tuf|X?(5~?nd=sFw6bpTD06|31{zZ8m0%Wx%$ zw<g_fOoc91mkcW;KeSP1Q@GH6!fAfG^N5MyT7`<Jy2Lc3d<SiD)~OR+U0XgFirScp z09B3fsH&i1e0ra+Q?Gb5uG~dIP9T)Al1Q@YlYv&UXrfFx;mJ!0TF1h_bej@Y=i;fw z6Cw>WC1D9^gI^VlT<A*y-BMLY8*?dDK2s&w`FeE=B}Z^=COC^5(x#K(dbTZz#Oe~w zKvylzC9{~>n8{5j_-p|)B(MQ%xsAc@(c5|si=sWAQBh^T0##gyGB25Yl3>ZmEfZd) zBGZ-^v6{`)j=eqYj9-9!nu4n1bwLW-pvH7fjBcr2Kq%SJs^e>}9sd>`DOlrv%bU3b zjJMemi+@SYL~S>7Hj7SY&}Mq!uV-i2wA>hV`&VopY!<IRhr4IpfXV7*m%17j!{RIF zK{Q4ET7WEJ?evRao*RVn#jX?2H$c^8!WuS(Y`@Z-f1F9Cc&&8cW;WBWxlN{Ogq`;9 zVC;ipMt=}oApXrW7?dUE*V?o6(&XM=ebPE=>@YY5mrnAzC39>-Ef?Drz5!UHcIQb< zsE%>&GFzwC2n>t{?31Gbld8!YFhW!)H;(a@qlMqo8q%WiimQ@Vql5E(rYXr%iQPdY zTw3#_kS1S!<mZ(1>ov=$0na656AEi6#5DhuPG472T^Wcc*$oqe7j|1G{5ru_gX&zR z#j??<Pmkb;GG|78@b5R-9)63ykpxvahzh6(Rx8S)>B$!@G&`ZaC2E|DN}|5>ivKRg zSE!XOhC(?(KgiCg|E=O)KLU=2={NWr$${G*93+qgc$B9Y&$I5+1t&PIiElpmMQpgi zEuByUzT1ZSies&lNl0@?&YL$5wvKp<wHMWHTK~jIqe5Izng%B8O>hv`Unf3N$AghF z3X)%~)_O}j!J#F+mt?f;oNbx834@S0K|P=2Hgiqs^*uL5n_m*$>n+=4KDFlLeExVA z^I^9V<d%GBpsrbID7b0u1)&h+*~1rQ0*}wR)z|&2Z)e`R`~IZ|y(hPu%A)vge4u=i zUWk41-o0rR-S3Ac4+_-a0)-?99t`IXV5NRWH%e<Kr=E!CLY;I}YJCqV__VPx*gWSD zvA0pwI)M$PnxEh!kj_TYohpN1m}=g}b-zCJ;1;m5vLJLI?m?$HJB~r}1UCk?%5c*I zcTO?dAQml#cW5w@G}={XnCV|QJ~;XBL(f|Z6tLPupF^)&LQP-hn?z&ryMZC_&}k&N ze8BU*YOnI<C2fX$9bl^FZ@>()0T|N~*0035OV6#lkq)P<)PNQ?23!OCWL$Y1yVl#r zosIPyI<pY=LwBRSElF(nRVKkt9pv5DG>-+u0lzV?hRoh+x?j-$Rdr#BlG54uaku!N z+Jp4Z>SAJR>tti=WbF748=Od*v{`3>8NLR7f(y9{KggrzB9?`hVt-J1*auWnj|rA6 zAQ-*9dn&$Mxm_a?&m5;qp^WGMl33Hw`DW!>mBdF`$trRbMKT9P8MAS?dwB755lKL3 zrELMDcC1VE*Q)OF)nrqLM4%PML^4{5W0gH9ikLM{0{bn|-@iezOZW@(7zie~PDyYM z%u4{B#&|RHp1=qq?FPMB6K@31bZfpxA~KZJt)Do<M+o+rzA=+<^R_G}S;aoo%7s&p zfV{DJ6^{`cw2@Gq-mDu2@hhprdjw0^x+q#(^sHDER!^vSBR{Ho?xqHdVBJu}DjM$t zD^(|pJ+Sc@mJWP}@-C8Z<V5Gy-e62NO*|5VST_E~c1j?8cMK27x12TcsDz&nT&)AN zWzxO>bwA$)JcBFN#Zp&P<=xUZpb`JBp}{UEEi{7r12}rbh8>N^cc~0QEJbfQ@u{Ay zVl+9CMSX+w0b<U6n%FHs^UugA%_R<QhFdA>GpGK=@#LO^_q>ObrltCw#_%(_YvNxx zhvRip4mx>4u3IL_4qP?tJmvNJ!9lZ)<-bNcVgyc&MTTzE7*0=OU9KlNLNfSE<UOLs z=T{o;Zkp8Z^irSlceQooqH3K#oxy+a8_3ik`xAb+01pZPfbd_rVC=yB4<-~PZOir( zpbTBhc!h?%N4w05WIF4#OkPaA90SJ9B}3o<H8|T|n=jiaA~SEw$Wb1@XT!_tyFBMB zD?uNVrbUtLrwRAjjTEmM9XuMdwh_j|PaH`5BVw*X$z2_#qSuqr21LqRd#$PE#4~|p zDxA4fvfLZewgSV8U^vd8L<l+vVB2wK4@hius)K8}#TH+sm#35S%6=F5qk@q`+hu^_ zw81V;bgH@pIgJN+B}#_S*6pAA`9~>D74aH0urEl?W#{w3K)Fh8W*Y%^%=HCT5jhLy zgEMp`J=H{g&)L=C5IovOmNjDCVP$G%F9!x2!iE6fpuE4z7A;P_UJsD)<rC8yRC5gW zEoTHGPZn{ad@V2GSlc*!viBTt_t;EXP6Y;z;hEj>tgUrit6%1S0aw`G-Ohcuo8(BD z+=5u-RI!8Mcq*PiXrk>_iCx;t9J#Holu6tsjO8*R4L6UEI@OZ&mj+X)75B|bm%2G= zyTNIaGBlb{5mWmS;N;(9UNWz(@ZNG|`u_jLkLlm|{a=kA<G=CyKOaAkIU|byRzJ;* z^^J@j{!@Q_4F9Ws7C+BKQ_KeU2Io1L3ghRw;98z^H*AvGUazhIuo42>xGX(-b>BCd zUK%~oipVpax2Qlygb+fd6>uF|WamxC9`^!6Wsy^n3R0)!(K?#3b#|sJ+2^bQEQ}2% z=R=NlvNI0F97!XwCuM0|Ivs^7NmbTIDTYv6EAl`U*0?jPC?4S?c@X^}fotS=%`Jgd zWJous4iC-OHk7z1VF1p<-g`wy->~(^jp2@xmDg9^AOiCgY9!ghv=Az-@jkmEag>1` zO|qpj7zc5i89Us-NwKGdCA)`H$k7m{6iHQ0Ke=kdo1iTx!143GDCLev;XW=6|GiR3 z%xX}5Vz}!%ITcQ#g$t$Iz$%uBy_pM+@d=vt=sf9O+r|Kn)g9BMl*wDcw_sYcMuty6 zscmYKj0{XJKj+q}vX(#(7?WoVc;O-cy<`TJ8V&gegRcC6S3v)z1ta<xdcixRJ_@A| zf08AFeLX078w#wP6cN(Hn8X-u_Y|ZV)7dJ=E?dU;$KC$B|NU?9`%nM-H-7(D;`g8R z^WXUWpNpSv|0(6#&--WOPcQPHD;>r*M*o{ugzf)rMH<$D%j$5TlZ6Em_75+`d`+zp zizbcH&m%u><>uzzj$&s=b>8xNj!3kb62z(V<Xut|x4V64x)Ml)ZJ?`q$FgvY2-q{H zbC+butn~@U;t2XeD(#8Lm80kiS;8c-@KQt};Q_~5Q1mF0ijfl7cA(n~D9=r%0^<nv zI<7aa+zEiEI!gQ#(cq<SLd3M*a9KC_mu>u=h6_CuGzNj|Rv*Lsvlg~YfDITSEwt-& zW*DOIzOq_6$I<?y5fT1pBa$^FX6~jI9rsyI@+2DT5-n9tciy{n4;G5)7V}Zy%G9Rn zlaW@(%r!?u7VAjzf$gYd&{F5P1D||U)^eKq45&g6XydBx=;KkgDXo|~dv$R?U}5{j zttONhs|cMM?q1o_k8LLG(p<munkXu=QU_9z)6mVjIzcZxXLP$WLm)oP8@mH(Q%*!4 z*--D;GJyWqZ3NPA+eUSHeSN85eQhjZnX8Mhm4EGw;JFJoU>^vy0Fl0MVA#JAPNR`j zQ(Go7v`u;993seO$;sgwvvBHb<QZ(!Rs8ht1;|R7)n+GUEIqUFw{d~3Bwzdk4}50# zzbwS}rjTEsxIc&3008j**@zfh8yFiIncJ8;(%V`Y%NbZ08#*!48JOE#cz|jtPcC1P zffj2D#vv)i)iC^4A`yW^gro#j!a|HgC=x(WN}@AIEE4`D8F!^1*p?}TYjCx7>}TM8 z{l4|Sx$XMh{Z8jLb;0X7%~Nl(RGHrSR8>)Y5hHGV;qQ~6h=A7s7e?4e>&}7Q0|5#W zZ<SRI48T_3+kJ;mYRNFNW8WX>4<85s#&oo<htNw9Qj0qj6K%f*wE{vQiJ`<$ra#k@ z7c2lChzAB7m5qJ)mHf?q2xSEFoi1cG<355QD??Z@S^m)+1lN={*=9#LKu(YXA^^HF zj_#ux^6>8ba&;ev;{=-EXG9phz4?JI2)MgvwRl`D^$TH67JNXDaKJab_iZ~kDqkuN z&FDrq<vm-z_ah8mxZ$h)BTU@3$PK##$?kePNb^n(bXUwRzV%)1Vf*1+eEs^@%4<v* zMij8-6>JcBH>X$pWyb{QflbkN+^x{r3RQt!aO{pO2ZS!iqjh|bq!<IDQvodgz8FQ* z0dW^7r`X#bLKHPalzol0;Ac@JeUt1&HBS-rX2827dXt7o%kfw1MYA1qo1ioDmA$t( zc9j?^UhuVPK-$tTd2Kpc6q7PNzZXW~DG{y{+ohS4_HClNfl)Y4n>&)m1Dd?xsorPe z&wv+U;SCI$Tg2^WfT`4y`nnOWs800Tgn%&>G#&5w6cPOFi?b8lheCUjT_lK_0O)vu zlAF9Pzw?#KYWym2N??H%CYjc-=FK=Bjj*{BPt8*HDnbvUWtX+9=3h&eAY9K3Rk(X= z9AaFkl%Y7p0WVeu7??nc9I&N5Z%)E-Ir_m}x%Jin4gi{>o4XI!G3=ozwatMHeeuW< z*UmRqMG$s7x>4l$D>R+TJxAZ)u)U<j#D#~4yJ)R;yOnKKF&X>B<e6z@BYsezpq%7_ zf`S7d-*TYf{k@U9^6l=5{j-aMbYyI7MFn+x{R2C{*54kC=zmSZ`GYP%sHNe9Ix+|L z^zZ+k42DL(>Nm#x&M-(cj1A+M$S*BDJvp<$M@Gk1`K_$Z!N~Gr>}_HJFfkqNH~7j} z)4~9lf=14$Hl(+Qoq{&ZXnlL<P_C+~3eWHHZ~}{jgk%){{TGMVL%yVd$JfBE#nIcv z1Y=1brwYH^6N`r4x_55@O|SKPA>_4Hf9Ld6tF*Eb$L{4u-<gZ6jkl$xWyIwWz18hD z>us+OMonPa7udrbPC5$S|CF+bpS*X)j2{E{6tu8^Qx9b5;ea458?!g7&_t|v4yf<F z6BeR7GEyi#Dk_Q@A3v9dl$7+`x3{{fqXS@RS%o1j0|&g3b#Z6>m~Cc79-fMNOhC#J zxvYu`7!<!w;Qg3PUPS{U0?P51?K3(9H*e7H_8L4y_us$DF=b_CgxuWkkS*V*c6N3( zk(atR#9DN8RRm;YV1Z#_5DfhM@K{(_euag^;*wFIe0;IJ^QCh7U(?;YxdSWV<>mSX zadB}(yu6=V(e5{irNa>|Eop0geK;G}QWy`9Pk!vSTYe1<%{z+~ssvP2P&D-Pfk#Is zHz%{i9Q>$7MIC_d?yWZ~()~{yx1HaqUH97^<)27PpB$8wr?M^A&3qXj1f_Dh^5>VI zP+;J1jAj;A0Hf8qfp2YGH@Ak25Rh<S{r&xUMMWSlZ*P73b0A$V-hll=odBIY;Ok=- zuI*ZV26#E4R8(U&EG#S+ARr(kE-&bt>FMbrNl8fs($Z0qKfvzz$P5tJ6VuXBfqZ&* z5)cpoG&DB)yIybgE+g%HodWIG@EWw5GQ1@FqKu4=ZfrMMW7lba2WRHQzvhL9k5>Nu zOZ$SGm7OghFOR?jH`=qfh^nNd<acmjv~#-aUprYli&zt(143<wUDL7h8a_0kOw--) zATr?m-6P{MPG=VyZb?z9s<e7V4(Zp=jc3<33~ZCi(X)>M{<Jpe|G8BAYikbvNf(cR zhX+SkR5W04P-JFnE5=-E>fOrCCMKX!nFZ|RP#uQ6uQMEU0i&|}U^4fSY}`U9G58T^ ztjy;Z?gFgY=Q?1gDYc%{58;;<^-|S2SG=1s(u|+{9v)b3?heOwU!<h?3@%W1MUS9| zbZMsbYJ<PWP)~JV%I7p^wetJ@I3Yn_5{G(Eqeg9a#MAl?99vinUckZMdQ$0gdlh|s zXO@>kYr7vBSloQr0v3}GZSNn!zoOYb!gJXwP-$OmkmL{P{S+~Ad)|eB!E{<1{TMxg z5~oq;{CDxOaC%NKLC?46=Yb<4B7W)E(50p4e3WnpD8qbTz+3;Q`kR(t$OZe&h!+Zh zl_sG7aeqhRnglGiI|lGB+Uu&J7yo6ua~ud47ngvQ6|||}VFe!kd!QR&ov`n_6V`Wc z`}K=?t;HHkoJH2((HVJ1>5&7W90M4fKN{dGLBB_n1te(B!*>TB5_0H)zrgpmesA?# zHNl4?f5Wca$2z}DA><!&z}ZsB8e;sV*G-5#NnD@?NMJw0I(!E3N1&yiFZ>tiJAS<f zISC1W>uQi3PyW-`n?l5|Yil(>@4f36$~ldh2f26sD##ubE+`jJSiq&@Pru1(@Qb@M zpmweGUYq^|{1@F0H$X6OZ~-45zM-emQZk?E?&6<|FP$<IxgUxK3}HdxKv;*Q8({bD zzbZnM<|~yOhyQ>uwX~!j8Jh@t??%}7^f?s!oB=bEdBNq!<@tt3odUR<1GOvm!tlb( z9n3Q}!(m98Ft}^g`Jvu5{B*a!KcHLZDDa`NaT=?Ejmw)Rwu&km`%8B-;A;2B=+{&O zJ<!LrlwX@O+D&7W;VchWllRck^(gM@&UNT=x-0@ZI&`%VKI6Hdkr9YlsCVBI-%C9H zdI<lmBC1nCb<;FZ`4J|84maq)7<tcW?EcP6=x@81O$MuXp&DJRz8m(1W&e{CGX$7w z-^9ekyxLmC`Y%&s<F63>RrY+nbs+}u=V^?%<j%5pX8uBN16s?MW9KGx<@0v@<4Q-j zqe{ot<I4M%L;ubxcfD>yyq)H8P`FtU0vZ~appX!QkL$U)IreW&wK(`{`ugsl#R3j! zdB;ROFDnEVcIB%2`g*9(3HV(^B;<s-dOR-oTet%By5?701RH*Se!t4`(a~x^sNlXE ze%~*gE%XJSdfv)!ZNF!$!o1RU;D$yvzl;pIo<2ePBlN3WVM)oc)%A5x%q_)ABF}Qu z=^XthrRf~r!J_NBI)<m_<a5_FvB@6Kfa)7OV}N@0=vZho0wSUfFP?WPL}cVo{qN>* zR+yRX+S)CH+NfTtsL^mlH~OFpF_T?y^CJXcVA^%v1RKPQFGs}6>S}{qA?&6*L(W>Y zfYww1tgClIH~gjTWylQdNK<z=j^OW%Td~)v?x)!GnvNi#&6}3xr2~3A9tyjw>Qx`q zApFnh@9RduS#M`!$fv>m8o3EA4yTizZ&^KHVhkUa6kls=ROYLS8;t5og3oq{X=;>< zY117}Ucet$(Dz=SIW!IyGPgxO$LHh~-{|yA-O;W8P@l?1?$ptIH)mpJucu`rD;Sxc zrXHQT`f~L4#tp4dJ+#f`_f`2VANS2dO$|+tiAjkR{?3{8&<q;RYdgdcmj>bx#e+dm zE6fgoy}SnN9c!u2UR`90iYhVfV5=pxr;vHuBC{GrzU3F|oZlXY_E7;0Dfkh`S_F@2 z$PpI`Q1mtF#+g6_Z!{uAm_X#7$%yu%KMJ_oLd@tOOwok%EjM=bp6P4&@a+o)$Mfd4 zUi<Xrc!5{DE;V<&dce|XYq`}&_bI+(caKAQw{+mj_Bag!k1Y44`eh0N-UhPSJJ112 zpe~Ng6xzAYdy;Csg&&OQvZ;A_`j1yiPMFhg(c@>i*YEa2{`7Am*t(aqt;M-T#DnG~ z_K<;?a;l8ejEwRO%dOTzTN9%)58365rq=k{Kd;(`%IexuO_`#uPEInbZb5F!%}GLJ zbpRjlmaoi1L&q$WtjfjF4XoE?R409lR+P(d)jJLfZZ9K5Fak74JE64bbwLKUe103k zz|F}JM3MA-r3D;q_>=)J#76Kn_Q-6r2khOpbjq~El>0uySon&(O(dvVb=ckX`>~{r zO(_GL3JHid7*i)q^{AjA?!OJTZ>?x4gD4F1t@3}}8|$O*zdpaD7Ty*XQgT8Tq`PI? zr)fK2gG@x8FW7xzt$?g@OJM(z;Nk`aDfx}BqH%;~jV3gYHp9AXvtEtUjhoVqfW7}T z6XE>!lz3N-<mR#*n=>>&J0yaR(yf8uLT*n$O>k2@ofOfjzuJto+9|sS-m;w5A*{Bb zX4cGG=#918@)-<_5YddjW^PkjKQ_B!I~vWFa0N@uZfKG0OCSXj50Id9Q|{ZtLpvxq z*FY|Lv(XuvtD}u%?-1t9%`|uOQXjX$?9uNLyt-1dI6X#Ui|@<!+ib<VJW*O>=})de zD9(4uMg+t3JZAur3~vXXZlbIq#rj0ode#T@R>P+AZfb%1wXh;Bjt8F=yUYIB=s6|z zX7oD?(N~uvFEH@i18o55z6nIc^`Ca1nE=C(oz@=Jy1KDxo@!_HFj1w@Bzp7T#%Pj^ zuBXCXRgZV~{PP?1R}Rx8@VfnN=1Yz83lXEsdAySGd8_J?;mMhqn(cg#VB<T*i>lJ& zFePr)%N^C7JdTs|#<jHT7}xsCk1Zf*U3&-4+}Nv<{ELJ0_%zATW_&Ij8rKQ&nG=Py zIJO<2WngBF`$b-1CQ9v02{b1P6dCT4Oj$_|bht-co;k_skLouO!j7EoV4E0!J`@iY zpo^W2F)amCNF;PFRVv<G705|QuPZL(N(Y`P%biFbeKPyjkXyck>~%8;$K+N;QwPob zXq<Z}WqaSf&S|?ZDmF^m+&NZ<0M}neutS7ON5zL&S+WHU9JgQ+wx#C97H4VX^D*`{ zC*gvba0p{Cb=90CNk=sp45Oqt^N8%Z87r?An4_ULBa63kw}ts3eLedQUEpc$C)~Sa z*SEebu%0%q$mvEU4Ut_1>2w`ZS<I|cnp@ai&|FFL7=s12cF$^s6ya}jna5V&cKF=& z%P2D4i3dFB(uNe~Dvfhr20bEi!N5%)j>D+7GmhGLPb-liD)PN_mbdS24E(eGKZZ5w z-+|wC@pc(vD^m)ZQmxDzcgGfaoYX(_S`x2x0AB%X&&@hZoecy_;F@)`Wgk!&!qIn9 zlDrb)rV<dxeOoHk_RB%IUcI84-nqbq{>G70hF)L}4a2gvkUy$6reZi1L#j1FUT!yP zW^#3Sb6HT%Eq9RFD|yePt{xjoIW;vY9SNnES)XTbV40=?6}6=jl^nTFcBsNfAIBuj zmC-sK$0<D9c^Aw|z5RBSX^x0Y@cd#r?(;|8HOo4q+5fx`dAcQs2XA0pQan<mA+zFs zdZ{>ypU8BRG=i<;uq4g$zzi?vXq{Fu;V=q>-g|N5m+AbrXyVVeeD(9#+=$+Xgbp=9 zdSB@}avI5wqv-k$mg(;`YA|($*{<8YF{WZR{qvC&K?ur->w-xaz}vBmsp}}o@y30Q zxib-h6DQbAp2#nOh7?{I$BmbK8Mrqxv*@C4PrLVWvz04QpX<o57qGfZHDu^~(Zqh| zJ;iD~zIh3KRW-dR2?-rz$j2nwICkoFE1pGQKK_K|O;w%TYrcoLMXIgI+?aMp)fy{j zKjc1?AUabGc2M+;fsboMj$`TKnQ>ia-!sJ&mX&lN4dc^UV-<8$IA&sDefdmmvftTM z@`_WW`m)6Aa5#dBBNE;s=!GG8`vg|i1>vejWj;=g^$_PZXmA%3vG0)}xxa}ac!r+a z$dn^3@^WpZ!;I(*%j~H|GS>Dfp4Zyp3MRLo+O94wMz!qZEV~@Jez_2aI0?hAekXZ{ zshXCo`vRfl({FHt{J59zkE4_wq%gA|Z(K_as1d4KiJy=&e2E>tA++qM1ozkDFGb?R zy)`TuWJT-MtpNG(^ht?p0=IqD3${Cjq%X#2Qoe+;;mV=|KP8!}V+3&Sp^vgiYTc*H z5V;659g|Hd9U4ms?!n@Vmx99SnGCarf+nBguJeZYs$SbBeO0t*QN^<uH!j+?Evv@e zsLIP~rnkZ%%(>D20-EVFJs+!ZJPU^yFP4EvYi0Qxn(<(gqd;}^h`f#&NyqJy#-4-R zi6f8BBsPyC+t-!F2EAo$2GVw!Tg&8VvsJ5tz^K^B@(bN`6;BF-Z)RS5`9YNpy|M}3 zKuMh|Unk}m)`r>{c2TLo@FSNn346?-`}C%y3#vdtS&|rMo9hGcq4z~ob`maJZDRrE zUnd{hq<K0~Nu|84T@(Ych(Fx|<#;_whSo`sq$FVs!#uYs*Q32{xc-dICUgv-IPI$Y zJ#q9VAtnt|tE|~1t6sdnZ6{&9x{Qxsi#5O2sw9)*<iAdWrewCwLlCGUX)+BmEKQzs zSEaMO=br~>1J|(W19;6ecVz7Y8zQ-Z0}d<rnp64INzzZ%*{dkwTK4nhebxtGo$Y(2 z%PIF&e{Vo)zV#niU95IWaj!FP1Oy8_aE&Lp35)33Z07O(xi%tvMEDB$_Pm~noO^zb zti}!B%9n_17j0K6;w-(%qdCuTTNv*Yd1x1|1WpDjZNV1Xa+6=vY=T5phSSVNETSxI zDB(@gJ~zru)|3(Q2>$!*b6Vl}$qlmJkK&;M9Yoc+S!CEaZ&M@+@oH>y-ixZoFPKUZ zlPa<<?R%l*!S<z9tqpM9J{x<>;#FBsuk$cO;*oJsn%zR}lP)tG?gUTQ57(J)mg`FD zNLiUxN|ffkc=PZ)xPXfY6AfYJw)1ogFdx>NE?GBt*5kK91sfvbu^rAlaLp>OdvMC7 z`8v7gm?`W!8ca^(GvVvioPFANQ5-t`Y$U~d{=T=B?yM!kE5yz3+!=wN%6@HeWapl_ z`x98vFlc9^hLamtn{4^+I&`^ZrxW8HrC{Mlbj?!JC{35(@;seAf0}-_i*{}I?P-7I zP(0kvcz5v7XpMH5MXKg0RJSS#KM{_{m1qrs2^~&!deaH4qX%qs;JL4qZIA`zLt~|f zQQkO~zTS0T=*Z$0LenN$uXI-GRw^RgU<qB@EFR=<p5O|cQdi^LN2p%3$F<{kRw?FP z(&4ZQEuIOz=e59pfNCEBaXsR_FGy~dCPN_4(uU{VmngfTSs54i*K~MjwCP{5d~5dM zDl)q3Pw(g9Wu0z@_wZ_YCfuBlV0LVW^S=<*>Yy|3v>1C4S|Xb-O_D>EMLk@%6orfS zOs#ea0*ybeicf*ksiaZdkg8EVb<;JQ=?OQ0_jZV#^`gdPsaFZJ7Hb|>#|!Lz)^^Hj z&V(1H{yS1YmYc=Jt5#QcMwqWtb1t1&facpb;_?8ao5b<WxlMPLEilllF+3$YL9=&T z4wfTbs=R*b%d{k=Q6u)zR^++RIqpS7DF&&zh1-5VM0g#vc3MV8{j>H6ziRZpZ;39w zp?UCJJr4S+N_7S!39h_RNIRD2XvvRKl53?fX_(hWt421|wDWP&M;uyVj7#xaTY<YC zYOmQhJ`^l*GoiGJQXY(rE;R`DCT%JMG6)>&-WA*YJ4{z?jgl;Tu#AXy&Y99%+J)74 zSLdb`LP5~qlOk0=m+X0XRJFL>NewoIPntDHxG^hd9>vSb+%C0ON6s?5?nwQ}`^@Fr z`%F5Dy!fyJBJf!>9WsQXO87$kIShH^FZNRdkIUv)MWBI+&(nhR&LC4SiA3p+CDk0V zhkJ7@qJyxqGjG~g63RzT>r%@#oJ5`(*Ol_?!)#mXVzCMq({mxyb-bn~!maMIV-u<s zR|)N|>M6%>v65dLZ-=1yT-9=%^>-_cqe`rv6)||j#NkDjIHdi&As*$!cLPn?i7#tj z8q|HTZTo`uL6(0m5)o)&kz*H$H-nj5JDlm^hM>VX6?f(8rY_nhua5sBNG!j)F`UWr zCSq~Ls0ZujiIIF>#2`O6ju^vn>e7we>)13B{qe;+%fI|nrez{&kU-AEkHk4ubl$Y1 ztyOD?9;qtjUVhwF;!5;Hj>HmWWgE1SLRnU6P$|Y)H^dN%H5DOGdTK`(a|TUWO-40{ zEbT14HX~9tn>!}iDwSs#U(%0MrJuN->Tw3(7}e`WThCt6zLI)@O%9#2Fp{{Ub}4#( zEW-Gc7Som6<Jn9YFdOq0D$2~=4Dbh?Udk1wxle2J+O<eF<h@QZ9QeBqaw>31Ij{P` z2N?wWn*wLq$YEO@yNpvX^!(Z&%a~Msz4VK9oj=U%Hqp$*u)1>kqzQm$SP04WbNVRg zcdFL#ej(e8m+0wqhbfJ`=-wKIMsWk1yt<E!+47-Ov*V^vC;T4^jI1{T+y4F%v(%`r z=KGqeA}h7@H2me+)gJcLzwgM~Rcjz@uN%|QJ&6}G4w3XU)ztRhlr%RTS`E7n!BJtu zfLphW>&NhT_oUODQ7cN|SgtsRk9Me#)@*;ujQ2L)jpn4%&R^m~;}6%RRMVJASZ4K~ z$%km8u2Qn*;+-z7#)=V%mTFEM#*irqTpaWEie@G1Vvp`8*G%7vy!}H;`Hlq#8-)Dl z+<qF`h5f{OuA;M?&`*lLWl`A;^(EoV4a^Xykde4fL<s{2mocT6lR40-$<nqIs2=>1 zP6IGJCjFN8n2XDL^pf?bWwOIEKIu-%lZOoN#QMikk=Of`cY*G@vPXkF)l0Y-=4C9^ zSQB>PeX!qVZ!~re@2XRG1G@oLw{Vt7A~so=yXlv~ERroso|(3IwWqAB_Ch%<$%3cC zjf|U`xJz!)FCkBfKvI&dR8IxS{lUc)5|8Gxck5Ojp<~c+B_j_bm0}gj?U}mHu0>7r zhGW>6o@&l783Qp(C|%yr9&_2|bSCX+7z*4fWH*<fJkRQqlCoIwaozLf24$oYvTC*b z)wVOJuY$Q*i^?_j5){tYKB-`Q_p=~K9*_PNAArCfF|NFyw+8UEyjXTg{8!5Ht`+?y z9Boa`^6i;hLvI2>+*6sG&aNM0K_VxY216mE4iSHZfvstDxn*%M*hu)3&ib3C?<Mn| zE=C>5kDFatEJjFh@OPPDsQxsoz?zX|<woIH{hcf0^X^-<jNj)}X4^l(+o*<|Wf>;X z*|wYY$-`qAw)+y8119q6k1R1@zq>1W*JVv!Equ_vW{x$@K=hq_kfn^^*IbHk>3l^_ zy|0NCzPM6&EYt?$)wj9_6TEUCMe8l9_x~^~uFkggT~T$+c-xq1?+`)}#f9?8sqgD} z0tdt2NfFgxo_F3zWbnW|El9Aewo<FNAXjR=pfK6IkEix?ijhwi`N)r-Og=BSMV<`q z01=#yQ`1;o*4*%x-n+b28)e?>)6y(0h624Dla4HN_o<W)mpsvz1JkpjI!W9ZM==rn zTkV{NA?C>tZI~0w6-|BF6X@i8w1g_kb(E>$l3`YDE!7Sw+ZsX9E*EtZlIF-u$elP? zEzD{)V%o~Vp0q5ar|!P1{AX*mvJ$=j(8GNaEh;oz;-w8OEh0K8th6Hv=v?HblZ^az zG#W-}`(PI8A^Bx!0!!7DLH_YNoc?}-l8ge9=<=#E!*xTk+=RaZ&6utz?>qt0==j+B zBUwh2IosCO)YUEM$TElS53LaiEMa;ViC3GPp`6jzLnbqIX`5JI6&?+O;|qqnvihH_ zy~xgu&6}^+nqp1fr}_byJaiJ2!ny?LQVe>;VGfJKwk1e=3XX&KHf8FE*=q9X1Ae*; zgAP4@7d=*WMaf`&-gr-qj}^O>-ba<pVkuddNu=@N=ZApE3hROMNq=6kwadl{)1;F7 zYHsf8%XKz(VX_EtUa}@#>aoy+B0D>ChHFM2biv&Xj)E0&_C2)qT6`Npj*;Qq$i8Xl z1PgmMMZnwEaq)=1FLL_mR)Zs@87t_x-C+7n4jd>b>wLA<gZV3^-o}<~$@|LylPYl0 zxtt8U8Tbz~9s6PB_e_yvxg0*QhgL2K3e)4c91Q;1_zQ9U_hM~iS@7VAZ%77;oRmNl z=;kjs+r&5!T!F7v=<dfPV7P%wgd!s_bHUQPy2>Lsb3s=<zHc|_h(-{{S^iPG-HvGx zm1JIYu*a5$R|>%>47PbuyM^1G;eu^vuCHQWWcQpe8JJGkT?RqB4BI;i!S}TKo$=Lk z*FCJntTh5WaR>wNp(!8K-|lzaKLZi}BEiG3DMShMlO8C-1^__)=Z9k_Gh=IG`v2Jf zM-1dbUCI_&45wT5%oXDj_ah890Rnt529_42U_jm;=Q_p}eEO==WC*fL*{)Kloa(O! zg@P@wl6CyAvtXZ$1|{0$7pTwa3y)62*xop$JT&}{`<Kt{*z8(*oi=P{C*hzX9&`7` zdu90efnz|Kz7*H`ZSLk>!g4z#y&>eeJ2GL9)F%Fq9ilA-#BfNUH({II6m-gD8)d4w zP-B|u`y`SY_5-3Js1l+|5FmWu?KI`H#WN4mA(mVD07E@bLyjUo0q@DV^&F;arWeVi z1!z9x&<xgE3K%ww#Rnl-qfkhvAoX!zPC)*v#P%^E7%5O}+G-0v9piU{VgdsTNbB$_ zQ$9{@S%j#w5ZR!>fSM&sE1ebE7i<8<$QGOC_0uINA22^}e40roN<-|<XbsP?dXTY@ z$PVsBlSci0sgGr}o6Jj>_Ez&<SPE=5v>WWj$06CxK5Th5E4s7oty8BZ+73YD{zp)+ z>%Jq8{vmAn0W&6zU*~RU@stEYQ&k+h*Z|vUWIZs-l*X!}xn;(!X#3n+4*+@rk*G3p z8AV26-aXECJP0V3fNqgHbFxhb<H4k~&=R^cN{t<{iZG~*fgNsd`;nzTnT}3!@@F5; zR^RV~t!cPF)EtoGO4Hrnd)J}41>JbQ&mX<2HXStkzTc<y+diLjuu8@|_0{;Fw+FNJ zwtV0Bq2O>YF+G7y9dd?5v(64taBeUmoFH-jZoCmbLFVZO<dCGYH6`|W?NoI$%j9W? zv$b~C2A(%5erP*DI(XZ)D11=^YQMp6EKoFic&F4ep-W{{8!PM3ha^tW=!T3p0Efir zR<m1QT!twR?pSE8&4g)KpD}}QLP-drPI>u=1_WF(6~?PQ7tpot@&%hIEey|`By!;@ ztuhd`^BD(>_Y@HJtY1y8(JUN~47#OatVonVP3TgB)NMMVhM|UKqwOcgMl97t=t$wo z`<ij+P=&cjq6%$(6*<jrARtR^R$zF<GhcXGA+3w&pNs@|gmBUmyS|no?{YYwNhg*? zeTv5rh%l}lkClToQwvWM!f~i$U1=t%?fN)dZx`hk&(R1xOsTa<2H3UZkz0j@>+7VS zFEYyLEVW<;Pi0GRv=71ly<2UD-^_l@wp!)Bd2(2s@G8^^$+Vrur2~4qay{1Ok$r+h z2?BoW;_SAa`hGp?sLYw}BAsrl;S7+2Ukj1xcv#>qio*Q^NxWQix{Wje?NMV!7E;D> zP>KlSXpcsRvJw@)bB6q<pdX@?pu~hEh{+UJGRXl_6}tZ7;4k~mXMSDd(Srh{S>=;k z9Bp+neI_V2GxTJz3zeA+vzqeCTJSt%g>esR@nV!k<Q!CD_S^*VGuZ-c-&<+Wng=g3 zY<K`BWWzQy*nCZvdzeuWaKf<odZ(HA<cl?E^J(P%*&G||x}vxyskpIyH0fT+EVoQj z35?0_IcILw7@Ats6O0CnrA)j#`?b3eyNv+-EFk`JO;i*8xjV1q%)2pz9QKv$l*TiK zKw8A|I1BV~ib5t$PDCk_x8+RO%g}npp4n)Xc?0XhC$vU}SO>FFJ8`c@BV)G7=NC0Z zdkoNFv!0<nfWR#-<=z}WNjK#*t$AcXR0|h96ICA_(EIM!gGGOJtlbXsQ>1K-wBE^< zncfM>Gt_#{&FvW!#XD0YNzUkR(~^^kVBjl_yih(IU=UF*F-<&ls=b-uxv)20KyYwk z=m$Z0G!4>$2l^EaRR@-HO-y)2c{_vO;4H!cf~AV_1V*}tE?oiYwAT^t#s^V%e2Qh% z^B0^1VnL72k5!iMyeKm55?2s`neZ9B)7gv=&iP#x<heA~I?rwl$Vs!+RFw-4Qq=*I z^W`@g)oYc8v|$zOsy>S)b}|BQ&{*uKe3{v{?;ijKu`3V4;RJ}&#^CTpQjFNJzmBaK z6HC;Mk-q*KF3^AkZ*ZEXM()F8YvZIIWqmr>+g@FG8NIe;iNUF=qDeI<w2h>#_VKHG z*?}8f_kEg8B(Xfjx7$%(N`D6e@TCh2wSB_>OVhANE^nXv`7eSW@_#W6YhxpGeR^|i zeN$sbI%8{-hBQberDe=v`ZDnX1OfF1B{cSj7L-UFUPgbaf^Y-gEAqcA)=A=-WoZhj zV14?m0a6GeB7g#b0*HY?U~wYyJ;B;4s9<TBdM1AQX|QS4&s*#a%nT^&IX<Re_gn4H zo$v2go!i=5^8ygwfvl*EO?Z~UcUdco@Q82RQw?i7X900=ciG`_fU?WV?V)Ljc!Ge5 z2zA4lxo2iatlINpmgi)|S|}O$DPR6~9?(lvm6vK_P7t7khyZ1&_BoV^U)R6DXO>SG zzM2KwXH3tUJ4<OFxt!hFYq=f7@J^_;rtoL4<Il!oJObt2n;;|j8t$u)#!9mcy?H~V z%DB20sGCN%L$g#vF>o?-O#`xSKZ~@ZlAg1e_}SF`&u@8Uw+%e{sQdj}(Fu`_nxBJI z%`7i!0|b3@9hh}vGr4m2$-^+_MM6$ajDqO)nL@@b`w!zx5e#z5hn9&`3<ILzD!eX# zAH{nlbgXVkb(ZW~7JPpHdLyy*zsq2ua*xSE&iu<mu5E{cZ)ieb^k;&Y)-YUYSw)gi zSN_~tSIqjjUt4l-5|27WYeGI*zk&zpbmv=Q?4uT}zxzA%^e6O@LDQt&GBjwMz3%_{ z{sHvyjfszr{kzTan9Y}$mlx^lyW&)S$Y5;c`09(idxs$RB`D~;cv=eoe)ZBh0uK;c zSP(8u1ef6II;cyBEsGELW^8cqki*AiY9LI1sDKrlaet?m{~Z+k8zDChLHljboiDfF zM*hL2S;$uxT>P#57tv9m5>Z~l?^`jF<eM`3{Coja8yg$O3f1Z_hqs4QlYM{H>LyoO z+R;PDR$G)l9l9!D_*A*EE5LLmnFJeF#*168lk0s!adExZtxiv392}etX&D*x)3Y<# z*`e)*(N!6ut6)I)@34b8zMXrHnR%HtMF8z=>zUQw09hMmC%s3A6@rV)yk#PNa`C~S ziaY<CzyPmEZEbDfR9elZ#gUPbyX~9jQ6mF+7+85{XgB$%Gg6j?WdUVlP#$_(|7F(2 zn>!K41707H=*LC*4;Fl=yL!LH4S+s8xWkHO;9XD8+z|%{2QDK+!+Nyl?(&!zI9wv+ zTS#gO3XHw|{c@X)7QoNXPkD89NFN^`J919IscdfO-QC?Z-RfZaJbQ<Sll+5ZYSk+y zW@gO-9PmODk0CZ2t!_D*tNu&6VhiKOo=(qa0@_Q{7Uy%cfx$t4X|lkK3@iu;h@R7D zLe@vTpNPtS*J<zVh-zcE(x9vB_Zd{x`!E<7SiSxFxw#>-qmx#9?4Squ#S`0~vp<sJ zt)A#qW8?WXH;`K0F0d_HbSS8(y|4Gj_T5VT?ai<~hc&<G&iiYLlm?5Fe)Hpn>Uz~a zFI1}6Ta`$sGr^jdw+9CgW^A^*4!e8c(Md{5_FZO3Cs*?DQI3Q*sh|l93;)vA#x*iA z+3O(L;Is7)-H+FQR){`y?Q62G8ZMR1wkV6XW*hF?nx9uZxx9p3L&3m+h>wqN!B-&Q z;J}cSk|HD}1r?OdEva2yTm=ktfP~S*$Dm{8J9MgK`k>Whi!av<w7zMaUNq0?{e3UA zAMw{$GQi!`rvC%qw!be=xaLtRh?Fn9=D`T)f&(q&xOyG|85y{?ww54X{3o6e|CaDL zPRzr`t#2kTOPUHvnOGA_isrog=6l80wiSZu?9gw9hB4UKJ27Iy&^iNQLfFVK+>Jyd z+ZEKho$UA6EVtACfaZ2X09&Qi$`tb5P!=~tsh@Z&cnqyhPt5xDbg4OX6OF$xhAwR1 zyB7nIvc2oSFVq`$MCv^M;4Yw?G;L!W91`LI(QOB)p{Z&9!7DD+zhD0ojWQzMR}QRx z(C6ur5dFJBrwB0AYVh))kDTSMC4riu4d#e<YyK8+E|WB^-#qUzwENcF(C|w@0Pw)b zRLC|~ujooRFo}NQMqp!levI8$iTt}r5acZx$*ydtSB?g7an)7(CZ5S#A4@YZH$5Ft zMn=Z&+lMNqABhLvQS5e*0}fL6u;1-twR>SacHyzdRC#kpYbsPu_fAj79DRlV-0x6C zOHHfCKiQlH_UrL{+3Y>>4gKIa`tLWu?z8y|1KxLkpUG=dD$tDYP^fup%Tzk9o>wj~ znZ%_Y-Z~gsJWWl<T1oHpxVQ9qZbn9gVKqc-=kG^yFV5WxFNA&0-`KvQYnH9*ZrCQb zyQ;sl8Q2qmQ&QJ@s478^Lb`%md!zNd%fthHU+;|d>R013Go=i>OEasptB-yzIvw8f zFcP`Fherm48fnXK{PGM@efu=;^nSW?X$RTF+M3p=sC55ny3}!6txLQgypGIlKXdQ4 zo2na~&x0!Dl2prV2eMqp(M?Z>r05jdhSl<he&=@iC!n>Qi9<n2S7IrgTVpy9+6x$- zP<`irp*s}Wz0E4A=g^5_vIM2SYKd2vL6@d6T|_^2C4aYm!*vgagp7Q{r6j%_yt0Bl z6Lt*p$ToAp%ojDxKIKWgJx>ud$=X>{TZz4tded7Q2aeBw1rjPv7vl|=lgg2#tNt7A zCv_KO>Gp{X0V1!gEVBF^73D<hRcDK|yFz@{e>o4T*=)nbkimZ(dcZWE<`KMjhz@mL zuO8t}s2%~zZ$fD$ci5BAgQIqq3Ow#z*ZXV&C@dx>q@;wB?OUvHI`Wa3n;N-{zMJ+Y zMJkB!2<f*x7?N>z426w7o?Q59maqlx0cMg?!MFMVTaLcEZicKM8XOF$o~o#D6(CMh z@FA5-x9csiDt^IuUyX6S%9UT-3VCKhy}MXpx^w8v-~;2CWWCkFc{7pD%+yheLE>2+ ziB3PqyEquR`;e9%FR(9{T`coipwmI_7Wzza>16A{%~+uDGl2i{XeFrGy(MAz&0a}J z)hapmm=Yglfgsg$-Q)Qa)k56!Hs>1N*ch-7EX!bgmxH8035;U#l0Rma7FC!?3BYOQ z1;6fJ)7btcvxuDk_wr*~R(gR2)%S$|Nq`JhfcfgVPPG-u^L?8`P8!=Xh#r5pI~Ts_ zDiBzK2mSsgD~(R4(PGf|UOy1)a2)e%W`^MMSzu!r7`Vur)J-e=#%n8bm<9jEi>=Bw zyQ7FBH}T2QLB#L-d30&X@sg}0%64i>)|Z0XlWb&4wpuEz`!@m_n#b_)@e3(@>}Qwf z)5Xhr2<*Ab>ME^kL&L|TfsjyOy4l5X(KDN2(OZZ>mqeW7ok0!qIf_A-r8@DVE3L`B zMUBleU7#~`vim_h$}3p0O<n{2?JM8|?HlyhcI?HBgA<Z_9&wU@4v_fkf!B+;&d7)q zA!kCvt}V@r@o1{mhAHKEZa&ZV&lvQ-SUEV4uzJ4&1^{@4`Y*Ewj>b+-{|KD_WB9H? z#oBR=1L4bu`57O>mk-DV(wv7d8>Npmd&Xtz6h4j?Fyq8pg(>LcO><NQ26B0K?Uek$ z+a-51IVZX0?S@3ew!g>WFPVe~o3w|ta)v`=2FW)5(dV<oG*<xyqDHMmogv$+-RCjB z>iP!L@yz5?oZzaKN}|za)+jiz=HGOZCu!^w1TBlWfnD-Su!J5%>$l{T5Dp!O-c6f0 z*wFo=;@QL^u;gYG?@{U*75Z%W(MN|->ZPv%2o6$PWTts#+H?oPeSowctcE&E8!Mwk z{_&6<t^_Neg#poslV+Jv@`m=sU%EuHl+)lCGpO}w@Zu3_1ghG5v)Bcx8>VUP){OG{ zZd@=GpJUVC_ySK6JrRmr=6RkvK>niSKN-@oYZXagO3*(@>!<o6qpV|h>XxhPM<D{i zrhyrFaEboleiEJf+w$ici1wu!JZ@i}Cs7<Og(twNG-~$fk9BrtB9fMA2>J|Ey%1=% zld8W+B)K#MVT`2|pr3z@^_d6CVh42U7+GU3W&o7q1op*2t(ItT&Jex*{uF7ai~-*{ zWks}v8qC08vkbd@OrHA7-jHr(n^;e7mHhk4P7Ag)fk!4`_U5jS87Ahp)qU}jtod+P z?-%+~?<vV?smzt?(M<o2GmYZI<S=!PkYy1kgw<7$jWWwKbO;5}Z5;enRB*`z{x8@M zC)_u`no)}6Bbpd8L0ka4F}#izsAgoV7*nMiQ~K!XMrRz%KnI`v0DRP*r0EaT;az>x zl`KTnLM$J}I)awg9g|)_mJP}X!~6Aw+rtDvvK<ZuXeA=>PT2R4aKwnh*GPBpccwnE zXt3h<LH@Fp6D}MblmDl^^ZsgTY1=SOkSbM>B2{|tLIk9T-UMmVK}smnTPPxegpTwk z5Ftnx5a}SjH<i#45NS%J_b>W-j+pnnYn?yf%Ub(~J+rTS&)O@qCwb<Xd)Br(LH>LU zQ;hIVAE|VDxh{K&Gap}qBbVBKdr{Tc?it%9_-N{y?=e^y)L`!)_crLkX{In2Ur)9Z z(@Xi}R-nrweQwAB`yztbfQ0YHoQD%V$GTwX85r|EFf4VclRSh!Pq1)?x>2~@i2G`Z z#M}qW7$bzk@oD=CpCxwC<1+Qf&LOFa4@Ym&MNcW0&D=>Db>66o(8ik&zEh;XyMf&< z<vIO&kPeTPVDk=cJASj;UJK;|PRM<4tJ`TT!yD(jQDV<C^X9JZMrG*NW8m>g)-rSn zuk#qJv5*ARc)_l@9V#Gl)DGDV45Mt&`>DAD;`$C^=C<-)+l<z(9`H`hdDd!IM@vlm zR%dzFx>ZLvfuawv7mW8J9+RicHBZQX;mOaDIE{wE!?s&bH>1Y=@T<maB)9icy+xlA zB#DzM9Wsa$w7oJqB{Oy-cdAUZIe3&U@&w-*F(q)=BSA=V@`|qTFlX=B>>pnauc39T ztjHzTq&R<=BshEMSy(#R{Cqnk8OMzClZVg8Ps%MH*k|@9X^5;9k#iyhuMJe7ee|Tp zH)EOxeWCWTrr!*nZIu$QOyt)wR>z0%DD(zxO0Nb;>~GCda^%qoB|F~BwGPwnAb;;z zCVJ&~x2`cgw?`uHgZ#rzylT7avx5&>>On7|`a_BpW^h=+mEz1-g>Fvvoh+5irb1WR zbZCII0qN@8kw=ZC<weONZULQFyideCeIODv?lDoksv2-f^*0F|s|VfTxULg@`i-?s z#_lo3`GfVM>4J_gY&k-6SWE~-fnal%+agozW6`;(yv#0FrpY$v3l6PZ1JmwGP$^5C zRcKGeJx4o?17g`e{4g4u4r3z4z-Kz_bEPke9811MJWZm_kooCWbGV(PduEL$`5SzD z1&B>4@P5Q&6Vl$!W&_Z>Qo@jQNu#~v7^*o|3fW{AhKd?4liUL;#d?#xT$U?+cT@SW zZ-FjWl*TyVxeNL)=GaQ=5Di5KH$JpVFUc$fchHWJvWYi#S}_nS`%v=@F=k1xRG_U4 zvJ4Z6&j5xQ4>N}s<5g|*{XfXl*RZT(#gj)^kI=pS7Pra7)OB)B$rMQ6$R3ZwOJ9qQ zMe_A#wbYW?e9QSMqLgbTSQ`VvUjaP!O&YZhfRMl?0!i}9Vwx3lt33n__gAD9k{~&E zC$c53?FY!?Gc8_cWJ`yF5=OW|<0f7K=VgXO<s_N(Zx#JSFCImE@OkYc!_oh}+|(F! zmWYd7?uI}f<bMULy=^RYFO51$qirgf0>s!04GWUoZ{NVLQyA9sMCdg}<Vwk;57oA* zu?`fJ?XA%{mMZh@B?ATNg-#RUl3nYB_c5R!AmKGI24AD_YKOu8od8!5CkHQ&X#rL# zJTw))d+IbMc85DVm;8+5IVX^Mx708tfyS}396gJGLvE})hRSp_!QeVwi}UwZGFyXo zrVg!IFO!c3w!_;Tc%G))uBzb2aeZa;g7Op`y%N?(qsO|{**<A$fQT<rR4sQr80k_+ zJ4wRYgm67=iuyO%rnIM|ioMw&Ev+jT!3cDj`I&OkN@MwG88#TXknUjcBU~yebB%y^ z6ZjQp^<Dxx7l*EePtM3Dc9B7u|9*5de!5l@$j3H`9I=U!4qiw*B&|m-uI_vmuC712 zBhu#X|6RL~U#U#pwjbUtP1h!+sp`d^y>@2DQ0R=a{9<8@*#R0(S7U3p2F6Y1!av4V z(HOw}iZ#E*PmD8jL~X3j>ElH7`n<8+a~k4woqE%?rN7yH<mJ{`9M)IU=ZC}P71mL< z(tXX6EHF3%TStMEw{-HR0QYT8ziUk@LMoDd9#Z>wkzI5Lrgp+JK4%k;<(gB6TU>n< zU3sc0El92NKYb7WMgeLXIv<mBCn#W3zznRwyL&#swpFq>i7%y~u%r#|+GSWJ&yJWZ zqGK_{m%>1NvC3G6E-({sG(A&c9@OM5!{RGCqz)A^VkFc*=a)EMmMr;-{pOw&ua6;9 z(PN%3g%7t=)syrB#GI=IFsdlt^A=YGAw&f#-}kkr`=w-6;bH|5Mi^`Pmebf&wP5mI z%N_0JOp~sLUwH+FVfk8ed1;NZKXu}!=?w-cQ`uOJB<y$K+)sW!X=0nmEY3&Uvk<cV zwqY^c<D<6AVK}^ASki~8^tzXdU7_T;w&{r!yVA}ISswa@#=#4O*pX@Dz&7AM_u1$I zm?r6ERzPw89wlxil)buo*#4HT%#04^sI9NWhSx~8!@1@%&L01HQPnvhV_d8;R~ItZ zwu{sI&r2m&#`f;oB74F|<nTmtxhGh;Jo-5@{eM?L_5<VxlF)`Uup<cnrrap^+&{xs zfuOiH5pAo3g(X~OM)niP0z`TXUf+DjzQL1xt95Yy;v{j4C4;V5aPo1bY%gs*yCS;s z%euG?+9saWMLN8a{Dg5GV2?KD^}%NwuOMql9GHBH_WT;bgaF`F(i$x&bDX0q8I+qR z`kiR4^SK~}r<90e3CHx9$O4dd{u3VU%7pd|hTI+EQIcMG@PwqU7upbil2(Yv^~8vU z*B?V*o1NhrM=K>^j>0C@*&S_x6_)%TJn7y8?e=!aPAh%DTl$1goLoB5C~nxn$jZW) z-mQ)(c^&K#4(zQkA_RnFvILv>S<VNHA3E67?e($mh!Q2=c###<*zBUW{WZbp&35@) z(pLKP?Cm$7Rk8Q;&d{%{(aNRM4a0{q0o!)UC;3@P2g?Zi+e)1*tYQ`bLb56rAhqVW z__>o3^<GZMd$-^VvTxlTWV!S9rxa22$S3^Yuj#zFzJMX5ZC$Ph(o_g}#ZT)xcMo5u z|2Ru+W*NIr*OPS5OOAhQ9o0RFJ9O?&Tgr5!Z9rt21TYo)hAa2Ts*`f^xy@*rnJD9q zk&;u+9K$qgS-xIlI6uLuEMPz1KPl&q27k{Z7i>wTZJf)I2gKSieSaF&dvKCr;KHhH zSP8Iz4IFhYry*B-&!6q_wcKgqd9|995|6RGGR@tb%3bVm<mFNhp#)g1IKEwed+f>^ z1-Z9wgQ!TU{8ag%`I$|v_4Cw7J-uQ7rza<ciP*XXFM5tlI!+%$HXy6dBj>q%iX#O1 zC@l?Zs|y*c*$i3wWxE==-)q>xR&3Okxdz$<&}^7^tvOcTJJtxP#}cSKq2J87sPZ5b zZ$l?C2<8U+pTzZt4(f;;7qi8Os?Wg|HLXuWh7Ggyp<u%qioyQR&yU=hcZwm|b5(-Z zKRu`^VMx<zj7{<7U|lC0>RYcOFWGtb<h*bvv92WZ&3u%++P+8BO3=hpX+uAO+W@Ee zw_|rEefVsS<W6o5n+4|7-*KBu=lI$gMp6DUMZ&clE^zV@KD)BdHNKtD*<StbGlfAK zH0=F|-WPc(fF#H);MLw_RegdwGq(9fS%trO$GvvgZiW$Mk>=59B81*ouyg{4l59NL zZ|7vol+pTy;F;wkFf!z^l6J_R5VkrLek99s+8w$&ubN#lf3eeL?YN`Jy<u_|<l$vr z#lVxeVJ2X<K4XycS<Ugg0#1S7hEWNP*y4q=1x_@nDFT0J7mJwAiCOJHJ3@|5Tf%>O zu;k7DtkV;RFJGY45@1-k#)D!;tFV-ZUpArCemgpZHwG@M7~|PaQ$(1Yu(iD10M7F6 z09!d^BCxDGK>w~gKtskI5E=8?=R4IPqJSe*w|6q-iAEK@Z&g!H!wD5h*KDBj)upde z7-4SSnuiuMvpfD&-Ul8xyJqkBKSdUO@F_EOP<Yd4>nk#4<IeHQL?2LmSdI`*=1PV= zd+2kV+l`-)#nL*NR(hK__GRZl<6IVFSHWWUjtb?Y_)H~tgcfCS>dk4l?2Ba>rz3MR z%QK~`+&=dxvq^A!*FuMV+jdfQSw-pe3_`UZ*a?Nsm{j2iULPj=wCEVglGx7q;@(-~ z2rCH|FsrgIZmEIw2K;mzn6%WbBA0jInf{3B17$k7Pzj<Yqz%GHNMIXaW6r;XkEtY7 z%;@k?p85Sq1a`>_lIV;<&0;slhw@7wMk2aOUqsWt@z;jhKJzx9!+AG6YYNrL>ApD| z3-jqoGeUz%&j_!o<xA>UJPqZcjF&>L8Sy;0F+Jp;22eU6@2;kZbTD<%SytJq>DfSF z*5Az+{P;}Cx||}m$DC`<Cn`|C-?~ZWL|J!OZNu=ry^$QX^%RGyV?!?8H&OlQ;ffk` zXL;=ecrJ}<T$|bMBmI7$z)q?E?Pe>-4?f1AVU>be$2hDeHJejqA;O@52j69%+r;Jr z#RSz?$TYu1j2Fj=el+Q)rv_pho=X_>-sey&tq!3A7rB%NE9Jl+?GnvG)s}_pO<Vd> zm6_`)I=WAWbMpM(PT0J>NkzTjZ+K6T*-F$7<6{5u@VUl#o9aER)nR#mf1rn7Piq-g z+ug1NYgu`nezkb~1goOOv$h<&1PWZjt+IuERW=GXCJkh_k0Qcwf%zhUgVI>97~h?+ zafM0o$=ekBlcu^)Rg~CWd_o+;zD7ZmhX?{ox!li3xjof|>6}jmEilEvk>(S6PjRwZ zup}EoXj)#=h!<xysTdmKeY-Vbe)Ij;TEO}lo!O}fAg#~xL7@lcBu1}d#gzoY7it^a zd>jXp+gH6Ow~G!^_-zPc7m_j;&+>_J)2Of&p<yA~z2@e6)KpY3w5668ORZf&>fIsH z7iBR|NO;g`sqSUR-#m{j`!3F33=q0>NFEa-j!bw$*N6m&5;#|BS!P36%YAB-u5e{X z#(m@nGRI<5=GiKXa}N{?9oR~#!;L$p^;D^wtQF^s58A$A_)$2j$_&d+-Y{JS=-~_s zY!!$fihP&2lV$N)BWBdJPStRLQc&Gz5zZD=Qk+t_Uklm#n&C_!v2_Fla2G$Uzn`_` zEl_wO$vhnz$n50(*whL<H8(s)v7g44H#8N~p249J8;+9!nY$`AN7wiO=p>&_00{$S ze~mFIPY`1o##U?>ohN_@OU*`X?!3d%2Y9iz8=9S$H~=)iXYJXHa@M_uH_IC-ctIBo z&8~y)->QdL1=gSy3k?XJA6k4ybX%D$7n*M_7ZU70rysSea5*hRj;jL&_EW}z{+11I znO{fT6m~y1RP~<&FNKb8XVg2tZ|C6io#?$y4PJDxN=6`8hOTwRQUSgll00T(khUCA zTP9Re2ZH++-u5Dkfkz}?;CMGlTwhZ_x(D_7rL1d1lj7hG9h|MPRE@OT%p;0Vu2yR+ zm5`raP13&W6Q|911*8`tLqoxYtik8c)V2T&74b`T6vdHR>M7s-PdSoc#Q1rch%4wD z;Wa_dg4}34l85=SSW`u|P2%lE{5S3NsOXUduE6Cw;t)|z`5%|E=Z9`H<jdks75yr5 za1kL8<AikIpy+x^MWG`L!Q=Rk+~3HL)4n1GKu=Lz=r5Ir`RP$lm?3yeZi-J3jEYZ^ zh%Zb2)ga;!LC&j}+9+=Z7ENIfG<z-GnzOeHwhB6gzIv%UKV$sXbYj$1Z<<Tm@v!OY zH)sGv!UX^j-0R|xNQKI#>rQ0%uk(HkxP{@lxU+HZf=-Gza5<J&<Ulsn@g*7<AWWL6 zd;6=ZuNW!0Z+BqYSH;kKt8Vj~8;YUt7To3w90TBB4h6ax8P|E?B`kWM_7hu2Hd-RL zdD<<kc9O3dX<0Jjt${n0WIm*8y}pvi)(hfUWzA-IR|D6+5D%S(HCKlj)r<=WcWI5D zhP_xn!j-exCDZcr6Y#M(W3O%p>!nVN*xrY|i(Y3lhlYnSz|dJu93t?Nsq@+k!pWHR z9h6&D=%j9+;NI22RI8gHrr;Kle*z@DW^*6Lh|VR#Vh+9OE3<Fe4h=6dhcR-hIn>p? zQz69tRf`2A<X_a0)2l#&tRh7TqdIaJKP3q->+JzSLwQch(K?g3-8}7?GBfCK+z$02 z4Re{AWqBNZRU*a;qfn!o^rhlk2Ex)7MSkFRts7@pN!bqw9^9A4E3KeMikBI!CrHc` zfsy(qHOtSb4S4ocqF*KZLA&m$f?;m&by6nk9AK*J+r4njs|o+B8p%%F+Uvorq9o(} zA^Ah+=!b-<&QdXy_b;7ZuD>dO-sJR;N|-823-<L?{WWQ>nU6W`$OzWB?KLhRi`8Y( zAlHA<6bvfJyj}!V`DrB(eYg{uuB0R4l0hU!NdSssB^IUN(=&G4*#Kt0OgFo0I@@Ih z#`J8k7}kWQ$CNzHovptywg=T}nC-F|o&y{B_4pq<v8>TCze+EEysA$9b^XQqFn7(h zEuD**&yg4ak!vMYchtcGQ93zh2;sAV%@2>(=7m>w`oMk_F3#!m#{srM7lSlrsgdpY zlhd1!KsZ2rKxl5CaxsHzaxQ7Y`3sfvj_0#V=B;nj;imdc_FyK@2C>0JhE0CKC+Ee3 z1daJY_LYJ231vyTUR!cXe&o=R+ZLkJ^U^I+fHPN+m~-~tOF_6A`npR)RyLr7Ii+fX z>?Ffh)V)@Kv|RMIbk+Ryn2so5o8a~V?xXHklyhu7glYVr>h*jdzO3A`cEtu6W1MJL z-yGqglUSvR<!m5F5K4<?IBzCO*Lp*J8LqNsaIam!!}?h5Mrv<W&_@rxRGc4$Pn;Vx zpYZ+t(;pp!2N}rv=PgH&7ya@1hYd)yRR0wCa~p%p^cFJgb7@lpRPaxK_sbw5GJyN@ zPssmpdq-75_1^xi)qqSG`l)o;i5nG1HP`)(-yr&h|6#R@%A*>x{^oBX{aJtB{s;fd zp7r0;D@b>(OY~23!hcUgks-ND^v|4HR2tPv?{~{>ka@X3=}X3l|DL{uoV~k5|Fq@% z_w)+YAM`)A9My#EcgvyFzv#=BWT-bswXXV&m(l)(|1TpeR4vqO=HFVY62G)ibDL2m zP;=XUOVHl^C4rjhhAM$NGx}SCQT3O^<#|$69ChyGH!h|53%5n?MfCSf3M!8pCI8JQ zY5(F;V&<p{e+G+x^Jr+pI%sI9p<`6~&j8zH8h-B*{a3gR6-Nz${1))K|JT#|J0PN^ XdIg!h{v!*K0!<ES!t7@K<Lv(ch=<2J literal 0 HcmV?d00001 diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-handlertype.json b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-handlertype.json index 48798bd0eb..3e79e59435 100644 --- a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-handlertype.json +++ b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-handlertype.json @@ -1,5 +1,9 @@ { "plugin-roots": ["/var/cache/tika/plugins"], + "grpc": { + "allowComponentManagement": true, + "allowPerRequestConfig": true + }, "pipes": { "numClients": 1, "configStoreType": "ignite", diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-local.json b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-local.json index 93499c5669..b79fe642bc 100644 --- a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-local.json +++ b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-local.json @@ -1,5 +1,8 @@ { "plugin-roots": ["/var/cache/tika/plugins"], + "grpc": { + "allowComponentManagement": true + }, "pipes": { "numClients": 1, "configStoreType": "ignite", diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json index 8863a4d90a..5639e397a4 100644 --- a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json +++ b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json @@ -1,5 +1,8 @@ { "plugin-roots": ["/var/cache/tika/plugins"], + "grpc": { + "allowComponentManagement": true + }, "pipes": { "numClients": 1, "forkedJvmArgs": [ diff --git a/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto b/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto index e05752b73c..59fcab37a6 100644 --- a/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto +++ b/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto @@ -200,7 +200,10 @@ message StringList { repeated string values = 1; } message ParseStatus { enum Status { UNSPECIFIED = 0; SUCCESS = 1; PARTIAL = 2; FAILED = 3; } Status status = 1; - int64 parse_time_ms = 2; + // Wall-clock milliseconds for the whole fetch+parse round trip as observed by the + // server. Fetch and parse happen together inside the forked pipes worker, so a + // parse-only time is not separable here -- the name says what is actually measured. + int64 fetch_parse_time_ms = 2; string parser_used = 3; repeated string parsers_used = 4; repeated string warnings = 5; diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java index 1bbee0045e..5c7dca245e 100644 --- a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java @@ -42,17 +42,28 @@ public final class DocumentBuilder { private DocumentBuilder() { } - /** Maps Tika parse output (primary metadata plus optional embedded metadata list) to {@link Document}. */ + /** + * Maps Tika parse output (primary metadata plus optional embedded metadata list) to + * {@link Document}. Pass {@code primary == null} when the pipes result carried no + * metadata at all (e.g. a fetch failure, or a crash before the parse produced + * anything): the returned Document then has only an id and a status. The status still + * derives from {@code pipesStatus} -- {@code EMPTY_OUTPUT} with no metadata is a + * legitimate success -- and an explicit error is recorded for the non-success cases. + */ public static Document build(Metadata primary, List<Metadata> allMetadata, String markdownBody, - String docId, String pipesStatus, long parseTimeMs) { + String docId, String pipesStatus, long fetchParseTimeMs) { if (primary == null) { - return Document.newBuilder() - .setStatus(ParseStatus.newBuilder() - .setStatus(ParseStatus.Status.FAILED) - .setParseTimeMs(parseTimeMs) - .addErrors("No metadata returned from parse") - .build()) - .build(); + ParseStatus.Builder statusBuilder = ParseStatus.newBuilder() + .setStatus(mapPipesStatus(pipesStatus)) + .setFetchParseTimeMs(fetchParseTimeMs); + if (statusBuilder.getStatus() != ParseStatus.Status.SUCCESS) { + statusBuilder.addErrors("No metadata returned from parse"); + } + Document.Builder document = Document.newBuilder().setStatus(statusBuilder.build()); + if (docId != null && !docId.isEmpty()) { + document.setId(docId); + } + return document.build(); } Document.Builder document = buildOne(primary, markdownBody, docId); @@ -60,7 +71,7 @@ public static Document build(Metadata primary, List<Metadata> allMetadata, Strin String parserClass = primary.get(TikaCoreProperties.TIKA_PARSED_BY); ParseStatus.Builder statusBuilder = ParseStatus.newBuilder() .setStatus(mapPipesStatus(pipesStatus)) - .setParseTimeMs(parseTimeMs); + .setFetchParseTimeMs(fetchParseTimeMs); if (parserClass != null && !parserClass.isEmpty()) { statusBuilder.setParserUsed(parserClass); } diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java index 8300d8df80..0a302d9d24 100644 --- a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java @@ -20,9 +20,7 @@ import java.util.Set; import org.apache.tika.grpc.v1.Document; -import org.apache.tika.grpc.v1.DocumentMetadata; import org.apache.tika.metadata.Metadata; -import org.apache.tika.metadata.TikaCoreProperties; /** * EPUB. Maps the common, cross-format facts into typed DocumentMetadata. @@ -43,16 +41,8 @@ public boolean appliesTo(Metadata tika) { @Override public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { - DocumentMetadata.Builder meta = document.getMetadataBuilder(); - // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). - TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); - TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.SUBJECT, meta::addAllKeywords, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + TransformSupport.mapCommonFields(tika, document.getMetadataBuilder(), consumed); // Everything EPUB-specific (epub:*, dc:identifier, OPF/NAV structural fields, ...) // lands in the tagged tail, appended once by DocumentTransformers. diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java index beb833df35..0a43502843 100644 --- a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java @@ -19,9 +19,7 @@ import java.util.Set; import org.apache.tika.grpc.v1.Document; -import org.apache.tika.grpc.v1.DocumentMetadata; import org.apache.tika.metadata.Metadata; -import org.apache.tika.metadata.TikaCoreProperties; /** * Always-applicable fallback. Pulls the universal Dublin Core fields into typed metadata @@ -37,14 +35,6 @@ public boolean appliesTo(Metadata tika) { @Override public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { - DocumentMetadata.Builder meta = document.getMetadataBuilder(); - - TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); - TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.SUBJECT, meta::addAllKeywords, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + TransformSupport.mapCommonFields(tika, document.getMetadataBuilder(), consumed); } } diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java index a9ff5cb966..7d7bd41bce 100644 --- a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java @@ -20,9 +20,7 @@ import java.util.Set; import org.apache.tika.grpc.v1.Document; -import org.apache.tika.grpc.v1.DocumentMetadata; import org.apache.tika.metadata.Metadata; -import org.apache.tika.metadata.TikaCoreProperties; /** * HTML / XHTML. Maps the common, cross-format facts into typed DocumentMetadata. @@ -48,16 +46,8 @@ public boolean appliesTo(Metadata tika) { @Override public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { - DocumentMetadata.Builder meta = document.getMetadataBuilder(); - // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). - TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); - TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.SUBJECT, meta::addAllKeywords, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + TransformSupport.mapCommonFields(tika, document.getMetadataBuilder(), consumed); // Everything HTML-specific (html:meta:*, og:*, twitter:*, html:link:*, ICBM, ...) // lands in the tagged tail, appended once by DocumentTransformers. diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java index cf9bcd6e15..e9ce47d897 100644 --- a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java @@ -49,7 +49,10 @@ public boolean appliesTo(Metadata tika) { public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { DocumentMetadata.Builder meta = document.getMetadataBuilder(); - // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + // Images deliberately do NOT use TransformSupport.mapCommonFields: their common + // facts come from image-native properties (IPTC keywords, EXIF/TIFF original date), + // and dc:creator / dc:language / dcterms:created rarely mean for a photo what they + // mean for a document, so those stay in the tagged tail untouched. TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); TransformSupport.addStrings(tika, IPTC.KEYWORDS, meta::addAllKeywords, consumed); diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java index 855c384d65..330469ac0f 100644 --- a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java @@ -37,6 +37,70 @@ */ public final class MetadataTagger { + /** + * {@link Property#get(String)} can only see Property constants whose declaring class + * has already been initialized by the JVM. In the gRPC server the parse runs in a + * forked pipes process, so nothing in the server JVM necessarily touches e.g. + * {@code org.apache.tika.metadata.PDF} before mapping -- and declared types + * ({@code pdf:encrypted} is a boolean, ...) would silently degrade to plain strings + * depending on class-loading order. Force-initialize the tika-core property + * vocabularies once so tagging is deterministic. + */ + private static final Class<?>[] PROPERTY_VOCABULARIES = { + org.apache.tika.metadata.AccessPermissions.class, + org.apache.tika.metadata.ClimateForcast.class, + org.apache.tika.metadata.CreativeCommons.class, + org.apache.tika.metadata.Database.class, + org.apache.tika.metadata.DublinCore.class, + org.apache.tika.metadata.DWG.class, + org.apache.tika.metadata.Epub.class, + org.apache.tika.metadata.ExternalProcess.class, + org.apache.tika.metadata.FileSystem.class, + org.apache.tika.metadata.Font.class, + org.apache.tika.metadata.Geographic.class, + org.apache.tika.metadata.HTML.class, + org.apache.tika.metadata.HttpHeaders.class, + org.apache.tika.metadata.IPTC.class, + org.apache.tika.metadata.MachineMetadata.class, + org.apache.tika.metadata.MAPI.class, + org.apache.tika.metadata.Message.class, + org.apache.tika.metadata.Office.class, + org.apache.tika.metadata.OfficeOpenXMLCore.class, + org.apache.tika.metadata.OfficeOpenXMLExtended.class, + org.apache.tika.metadata.PageAnchoring.class, + org.apache.tika.metadata.PagedText.class, + org.apache.tika.metadata.PDF.class, + org.apache.tika.metadata.Photoshop.class, + org.apache.tika.metadata.PST.class, + org.apache.tika.metadata.QuattroPro.class, + org.apache.tika.metadata.Rendering.class, + org.apache.tika.metadata.RTFMetadata.class, + org.apache.tika.metadata.TIFF.class, + org.apache.tika.metadata.TikaCoreProperties.class, + org.apache.tika.metadata.TikaPagedText.class, + org.apache.tika.metadata.WARC.class, + org.apache.tika.metadata.WordPerfect.class, + org.apache.tika.metadata.XMP.class, + org.apache.tika.metadata.XMPDC.class, + org.apache.tika.metadata.XMPDM.class, + org.apache.tika.metadata.XMPIdq.class, + org.apache.tika.metadata.XMPMM.class, + org.apache.tika.metadata.XMPPDF.class, + org.apache.tika.metadata.XMPRights.class, + org.apache.tika.metadata.Zip.class, + }; + + static { + for (Class<?> vocabulary : PROPERTY_VOCABULARIES) { + try { + // A class literal alone does NOT initialize the class; forName(…, true, …) does. + Class.forName(vocabulary.getName(), true, vocabulary.getClassLoader()); + } catch (ClassNotFoundException e) { + throw new ExceptionInInitializerError(e); + } + } + } + private MetadataTagger() { } diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java index bbc899ba13..1dd66fb33e 100644 --- a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java @@ -23,7 +23,6 @@ import org.apache.tika.grpc.v1.DocumentMetadata; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Office; -import org.apache.tika.metadata.TikaCoreProperties; /** * Office (MS Word/Excel/PowerPoint, OOXML, OpenDocument). Maps the common, cross-format @@ -59,13 +58,8 @@ public void transform(Metadata tika, Document.Builder document, Set<String> cons DocumentMetadata.Builder meta = document.getMetadataBuilder(); // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). - TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); - TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); - TransformSupport.addStrings(tika, Office.KEYWORDS, meta::addAllKeywords, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + // Office keywords live under meta:keyword, not dc:subject. + TransformSupport.mapCommonFields(tika, meta, consumed, Office.KEYWORDS); TransformSupport.setInt(tika, Office.PAGE_COUNT, meta::setPageCount, consumed); TransformSupport.setLong(tika, Office.WORD_COUNT, meta::setWordCount, consumed); TransformSupport.setLong(tika, Office.CHARACTER_COUNT, meta::setCharacterCount, consumed); diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java index a39131d2b2..0439a0cc59 100644 --- a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java @@ -24,7 +24,6 @@ import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Office; import org.apache.tika.metadata.PagedText; -import org.apache.tika.metadata.TikaCoreProperties; /** * PDF. Maps the common, cross-format facts into typed DocumentMetadata. @@ -48,13 +47,7 @@ public void transform(Metadata tika, Document.Builder document, Set<String> cons DocumentMetadata.Builder meta = document.getMetadataBuilder(); // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). - TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); - TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.SUBJECT, meta::addAllKeywords, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + TransformSupport.mapCommonFields(tika, meta, consumed); TransformSupport.setInt(tika, PagedText.N_PAGES, meta::setPageCount, consumed); TransformSupport.setLong(tika, Office.WORD_COUNT, meta::setWordCount, consumed); diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java index 6090e8f255..5d29835614 100644 --- a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java @@ -23,7 +23,6 @@ import org.apache.tika.grpc.v1.DocumentMetadata; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Office; -import org.apache.tika.metadata.TikaCoreProperties; /** * RTF. Maps the common, cross-format facts into typed DocumentMetadata. @@ -49,12 +48,8 @@ public void transform(Metadata tika, Document.Builder document, Set<String> cons DocumentMetadata.Builder meta = document.getMetadataBuilder(); // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). - TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); - TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); - TransformSupport.addStrings(tika, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); - TransformSupport.addStrings(tika, Office.KEYWORDS, meta::addAllKeywords, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.CREATED, meta::setCreated, consumed); - TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + // RTF keywords ride on the Office property set (meta:keyword), not dc:subject. + TransformSupport.mapCommonFields(tika, meta, consumed, Office.KEYWORDS); TransformSupport.setInt(tika, Office.PAGE_COUNT, meta::setPageCount, consumed); TransformSupport.setLong(tika, Office.WORD_COUNT, meta::setWordCount, consumed); TransformSupport.setLong(tika, Office.CHARACTER_COUNT, meta::setCharacterCount, consumed); diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java index 541bb7e147..660ccbe5db 100644 --- a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java @@ -25,8 +25,10 @@ import com.google.protobuf.Timestamp; +import org.apache.tika.grpc.v1.DocumentMetadata; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Property; +import org.apache.tika.metadata.TikaCoreProperties; /** * Helpers for transformers: set a typed Document field from a Tika Property and mark the @@ -37,6 +39,33 @@ public final class TransformSupport { private TransformSupport() { } + /** + * Maps the common, cross-format fields -- title, description, authors, keywords, + * languages, created, modified -- into typed {@link DocumentMetadata}. Keywords come + * from {@link TikaCoreProperties#SUBJECT} (Dublin Core). Kept in one place so the + * common mapping cannot drift between format transformers. + */ + public static void mapCommonFields(Metadata md, DocumentMetadata.Builder meta, Set<String> consumed) { + mapCommonFields(md, meta, consumed, TikaCoreProperties.SUBJECT); + } + + /** + * Same as {@link #mapCommonFields(Metadata, DocumentMetadata.Builder, Set)} but with + * the keyword source overridden, for formats whose keyword bag lives under a + * different property (e.g. Office and RTF use {@code meta:keyword}); {@code dc:subject} + * then stays in the tagged tail because those formats treat it as a distinct field. + */ + public static void mapCommonFields(Metadata md, DocumentMetadata.Builder meta, Set<String> consumed, + Property keywordsProperty) { + setString(md, TikaCoreProperties.TITLE, meta::setTitle, consumed); + setString(md, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); + addStrings(md, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); + addStrings(md, keywordsProperty, meta::addAllKeywords, consumed); + addStrings(md, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); + setTimestamp(md, TikaCoreProperties.CREATED, meta::setCreated, consumed); + setTimestamp(md, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + } + public static void setString(Metadata md, Property key, Consumer<String> setter, Set<String> consumed) { String v = md.get(key); if (v != null && !v.trim().isEmpty()) { diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java index 59c4411e23..7cff080069 100644 --- a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java @@ -50,7 +50,7 @@ void buildsContentTreeAndMetadataFromRealPdf() throws Exception { assertTrue(document.getExtraCount() > 0); assertEquals(ParseStatus.Status.SUCCESS, document.getStatus().getStatus()); - assertEquals(42L, document.getStatus().getParseTimeMs()); + assertEquals(42L, document.getStatus().getFetchParseTimeMs()); } @Test @@ -75,14 +75,29 @@ void recursesIntoEmbeddedDocuments() { assertEquals("an embedded photo", child.getMarkdown()); } + /** + * {@code primary == null} is the server's signal that the pipes result carried no + * metadata at all (see TikaGrpcServerImpl). On a failure status that surfaces as an + * explicit error; on {@code EMPTY_OUTPUT} -- a success with nothing to map -- it must + * NOT be reported as a failure. + */ @Test void reportsFailureWhenNoMetadata() { - Document document = DocumentBuilder.build(null, null, null, "doc-1", "PARSE_EXCEPTION", 1L); + Document document = DocumentBuilder.build(null, null, null, "doc-1", "FETCH_EXCEPTION", 1L); + assertEquals("doc-1", document.getId()); assertEquals(ParseStatus.Status.FAILED, document.getStatus().getStatus()); assertFalse(document.getStatus().getErrorsList().isEmpty()); } + @Test + void emptyOutputWithNoMetadataIsSuccessNotFailure() { + Document document = DocumentBuilder.build(null, null, null, "doc-1", "EMPTY_OUTPUT", 1L); + + assertEquals(ParseStatus.Status.SUCCESS, document.getStatus().getStatus()); + assertTrue(document.getStatus().getErrorsList().isEmpty()); + } + /** * {@code pipesStatus} is populated from {@code PipesResult.RESULT_STATUS.name()} (see * org.apache.tika.pipes.api.PipesResult), never from a plain "OK" -- there is no such diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java index cb2bad1a79..e6fdf1eeb2 100644 --- a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java @@ -46,11 +46,11 @@ protected ParseFixture parseBody(String fileName) throws Exception { return new ParseFixture(metadata, List.of(metadata), writer.toString()); } - protected Document map(ParseFixture fixture, String docId, long parseTimeMs) { + protected Document map(ParseFixture fixture, String docId, long fetchParseTimeMs) { // "PARSE_SUCCESS" mirrors a real org.apache.tika.pipes.api.PipesResult.RESULT_STATUS // name -- there is no "OK" constant on that enum. return DocumentBuilder.build( - fixture.primary(), fixture.allMetadata(), fixture.body(), docId, "PARSE_SUCCESS", parseTimeMs); + fixture.primary(), fixture.allMetadata(), fixture.body(), docId, "PARSE_SUCCESS", fetchParseTimeMs); } protected Document map(ParseFixture fixture, String docId) { diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java index d359ec1504..3cef70370d 100644 --- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java +++ b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java @@ -296,7 +296,9 @@ private void fetchAndParseImpl(FetchAndParseRequest request, } Metadata tikaMetadata = new Metadata(); - long parseStart = System.nanoTime(); + // Times the whole pipesClient.process() round trip: fetch and parse both happen + // inside the forked pipes worker, so this is fetch+parse latency, not parse-only. + long fetchParseStart = System.nanoTime(); try { ParseContext parseContext = new ParseContext(); String additionalFetchConfigJson = request.getAdditionalFetchConfigJson(); @@ -318,20 +320,22 @@ private void fetchAndParseImpl(FetchAndParseRequest request, if (pipesResult.status().equals(PipesResult.RESULT_STATUS.FETCH_EXCEPTION)) { fetchReplyBuilder.setErrorMessage(pipesResult.message()); } - long parseTimeMs = (System.nanoTime() - parseStart) / 1_000_000L; + long fetchParseTimeMs = (System.nanoTime() - fetchParseStart) / 1_000_000L; List<Metadata> metadataList = new ArrayList<>(); if (pipesResult.emitData() != null && pipesResult.emitData().getMetadataList() != null) { metadataList.addAll(pipesResult.emitData().getMetadataList()); } - Metadata primary = metadataList.isEmpty() ? tikaMetadata : metadataList.get(0); - String body = primary.get(TikaCoreProperties.TIKA_CONTENT); + // null (not an empty Metadata) when the pipes result carried no metadata, so + // DocumentBuilder can distinguish "nothing came back" from an empty parse. + Metadata primary = metadataList.isEmpty() ? null : metadataList.get(0); + String body = primary == null ? null : primary.get(TikaCoreProperties.TIKA_CONTENT); Document document = DocumentBuilder.build( primary, metadataList, body, request.getFetchKey(), pipesResult.status().name(), - parseTimeMs); + fetchParseTimeMs); fetchReplyBuilder.setDocument(document); responseObserver.onNext(fetchReplyBuilder.build()); } catch (IOException e) {