Index

Symbols

--aws-access-key fastwarc-benchmark command line option --aws-secret-key fastwarc-benchmark command line option --bench-warcio fastwarc-benchmark command line option --compress-alg fastwarc-recompress command line option --compress-level fastwarc-recompress command line option --cutoff resiliparse-lang-evaluate command line option --decompress-alg fastwarc-benchmark command line option fastwarc-check command line option fastwarc-recompress command line option --endpoint-url fastwarc-benchmark command line option --fasttext-model resiliparse-lang-benchmark command line option resiliparse-lang-evaluate command line option --fields fastwarc-index command line option --filter-type fastwarc-benchmark command line option --headers fastwarc-extract command line option --is-prefix fastwarc-benchmark command line option --jobs resiliparse-lang-create-dataset command line option resiliparse-lang-download-wiki-dumps command line option --langs resiliparse-lang-download-wiki-dumps command line option resiliparse-lang-evaluate command line option --min-examples resiliparse-lang-create-dataset command line option --out-format resiliparse-lang-train-vectors command line option --outdir resiliparse-lang-download-wiki-dumps command line option --output fastwarc-check command line option fastwarc-extract command line option fastwarc-index command line option --parse-http fastwarc-benchmark command line option --payload fastwarc-extract command line option --preserve-multi-header fastwarc-index command line option --print-cm resiliparse-lang-evaluate command line option --quiet fastwarc-check command line option fastwarc-recompress command line option --rounds resiliparse-lang-benchmark command line option --sort-lang resiliparse-lang-evaluate command line option --split resiliparse-lang-evaluate command line option resiliparse-lang-train-vectors command line option

--test-size resiliparse-lang-create-dataset command line option --truncate resiliparse-lang-evaluate command line option --use-python-stream fastwarc-benchmark command line option --val-size resiliparse-lang-create-dataset command line option --vector-size resiliparse-lang-train-vectors command line option --verify-digests fastwarc-benchmark command line option --verify-payloads fastwarc-check command line option -a fastwarc-benchmark command line option -c fastwarc-recompress command line option resiliparse-lang-evaluate command line option -d fastwarc-benchmark command line option fastwarc-check command line option fastwarc-recompress command line option -e fastwarc-benchmark command line option -f fastwarc-benchmark command line option fastwarc-index command line option resiliparse-lang-benchmark command line option resiliparse-lang-evaluate command line option resiliparse-lang-train-vectors command line option -H fastwarc-benchmark command line option -j resiliparse-lang-create-dataset command line option resiliparse-lang-download-wiki-dumps command line option -l fastwarc-recompress command line option resiliparse-lang-download-wiki-dumps command line option resiliparse-lang-evaluate command line option resiliparse-lang-train-vectors command line option -o fastwarc-check command line option fastwarc-extract command line option fastwarc-index command line option resiliparse-lang-download-wiki-dumps command line option -p fastwarc-benchmark command line option fastwarc-check command line option -q fastwarc-check command line option fastwarc-recompress command line option -r resiliparse-lang-benchmark command line option -s fastwarc-benchmark command line option resiliparse-lang-evaluate command line option resiliparse-lang-train-vectors command line option -t resiliparse-lang-evaluate command line option -v fastwarc-benchmark command line option -w fastwarc-benchmark command line option

_

__getitem__() (resiliparse.parse.html.DOMCollection method) (resiliparse.parse.html.DOMElementClassList method) (resiliparse.parse.html.DOMNode method) __iter__() (fastwarc.warc.ArchiveIterator method) (resiliparse.parse.html.DOMCollection method) (resiliparse.parse.html.DOMElementClassList method) (resiliparse.parse.html.DOMNode method)

__next__() (fastwarc.warc.ArchiveIterator method) __setitem__() (resiliparse.parse.html.DOMNode method)

A

add() (resiliparse.parse.html.DOMElementClassList method) any_type (fastwarc.warc.WarcRecordType attribute) append_child() (resiliparse.parse.html.DOMNode method)

ArchiveIterator (class in fastwarc.warc) ATTRIBUTE (resiliparse.parse.html.NodeType attribute) attrs (resiliparse.parse.html.DOMNode attribute)

B

body (resiliparse.parse.html.HTMLTree attribute) BrotliReader (class in fastwarc.stream_io)

BrotliWriter (class in fastwarc.stream_io) bytes_to_str() (in module resiliparse.parse.encoding)

C

CDATA_SECTION (resiliparse.parse.html.NodeType attribute) child_nodes (resiliparse.parse.html.DOMNode attribute) ChunkedReader (class in fastwarc.stream_io) ChunkedWriter (class in fastwarc.stream_io) class_list (resiliparse.parse.html.DOMNode attribute) class_name (resiliparse.parse.html.DOMNode attribute) close() (fastwarc.stream_io.WarcReader method) (fastwarc.stream_io.WarcWriter method) COMMENT (resiliparse.parse.html.NodeType attribute)

consume() (fastwarc.warc.WarcRecord method) (fastwarc.warc.WarcRecordPayloadReader method) CONTENT_LENGTH (fastwarc.warc.WarcHeader attribute) content_length (fastwarc.warc.WarcRecord property) CONTENT_TYPE (fastwarc.warc.WarcHeader attribute) continuation (fastwarc.warc.WarcRecordType attribute) conversion (fastwarc.warc.WarcRecordType attribute) create_element() (resiliparse.parse.html.HTMLTree method) create_text_node() (resiliparse.parse.html.HTMLTree method)

D

decode() (resiliparse.beam.coders.StrUtf8Coder method) decompose() (resiliparse.parse.html.DOMNode method) delattr() (resiliparse.parse.html.DOMNode method) delete_action() (in module resiliparse.beam.elasticsearch) detect_encoding() (in module resiliparse.parse.encoding) detect_fast() (in module resiliparse.parse.lang) detect_mime() (in module resiliparse.parse.encoding) document (resiliparse.parse.html.HTMLTree attribute)

DOCUMENT (resiliparse.parse.html.NodeType attribute) DOCUMENT_FRAGMENT (resiliparse.parse.html.NodeType attribute) DOCUMENT_TYPE (resiliparse.parse.html.NodeType attribute) DOMCollection (class in resiliparse.parse.html) DOMContext (class in resiliparse.parse.html) DOMElementClassList (class in resiliparse.parse.html) DOMNode (class in resiliparse.parse.html) DUMPDATE resiliparse-lang-download-wiki-dumps command line option

E

ElasticsearchBulkIndex (class in resiliparse.beam.elasticsearch) ELEMENT (resiliparse.parse.html.NodeType attribute) encoding() (resiliparse.parse.encoding.EncodingDetector method) EncodingDetector (class in resiliparse.parse.encoding) ensure_index() (in module resiliparse.beam.elasticsearch) ENTITY (resiliparse.parse.html.NodeType attribute)

ENTITY_REFERENCE (resiliparse.parse.html.NodeType attribute) exc_loop() (in module resiliparse.itertools) exception (resiliparse.process_guard.InterruptType attribute) exception_then_signal (resiliparse.process_guard.InterruptType attribute) ExecutionTimeout extract_plain_text() (in module resiliparse.extract.html2text)

F

fastwarc-benchmark command line option --aws-access-key --aws-secret-key --bench-warcio --decompress-alg --endpoint-url --filter-type --is-prefix --parse-http --use-python-stream --verify-digests -a -d -e -f -H -p -s -v -w INPUT_URL fastwarc-check command line option --decompress-alg --output --quiet --verify-payloads -d -o -p -q INFILE fastwarc-extract command line option --headers --output --payload -o INFILE OFFSET

fastwarc-index command line option --fields --output --preserve-multi-header -f -o INFILES fastwarc-recompress command line option --compress-alg --compress-level --decompress-alg --quiet -c -d -l -q INFILE OUTFILE fastwarc.stream_io module fastwarc.warc module FastWARCError finish() (fastwarc.stream_io.WarcWriter method) first_child (resiliparse.parse.html.DOMNode attribute) first_element_child (resiliparse.parse.html.DOMNode attribute) flush() (fastwarc.stream_io.WarcWriter method) frame_start_position() (fastwarc.stream_io.WarcReader method) freeze() (fastwarc.warc.WarcRecord method)

G

get_element_by_id() (resiliparse.parse.html.DOMCollection method) (resiliparse.parse.html.DOMNode method) get_elements_by_attr() (resiliparse.parse.html.DOMCollection method) (resiliparse.parse.html.DOMNode method) get_elements_by_class_name() (resiliparse.parse.html.DOMCollection method) (resiliparse.parse.html.DOMNode method)

get_elements_by_tag_name() (resiliparse.parse.html.DOMCollection method) (resiliparse.parse.html.DOMNode method) getattr() (resiliparse.parse.html.DOMNode method) GzipReader (class in fastwarc.stream_io) GzipWriter (class in fastwarc.stream_io)

H

has_block_digest() (in module fastwarc.warc) has_content_length_gte() (in module fastwarc.warc) has_content_length_lte() (in module fastwarc.warc) has_payload_digest() (in module fastwarc.warc) has_record_type() (in module fastwarc.warc) has_valid_block_digest() (in module fastwarc.warc) has_valid_payload_digest() (in module fastwarc.warc) hasattr() (resiliparse.parse.html.DOMNode method) head (resiliparse.parse.html.HTMLTree attribute)

HeaderMap (class in fastwarc.warc) headers (fastwarc.warc.WarcRecord property) html (resiliparse.parse.html.DOMNode attribute) HTMLTree (class in resiliparse.parse.html) http_charset (fastwarc.warc.WarcRecord property) http_content_type (fastwarc.warc.WarcRecord property) http_date (fastwarc.warc.WarcRecord property) http_headers (fastwarc.warc.WarcRecord property) http_last_modified (fastwarc.warc.WarcRecord property)

I

id (resiliparse.parse.html.DOMNode attribute) index_action() (in module resiliparse.beam.elasticsearch) INDIR resiliparse-lang-create-dataset command line option resiliparse-lang-evaluate command line option resiliparse-lang-train-vectors command line option INFILE fastwarc-check command line option fastwarc-extract command line option fastwarc-recompress command line option resiliparse-lang-benchmark command line option INFILES fastwarc-index command line option init_headers() (fastwarc.warc.WarcRecord method)

inner_seek() (fastwarc.stream_io.WarcReader method) inner_tell() (fastwarc.stream_io.WarcReader method) INPUT_URL fastwarc-benchmark command line option insert_before() (resiliparse.parse.html.DOMNode method) InterruptType (class in resiliparse.process_guard) is_concurrent() (in module fastwarc.warc) is_frozen (fastwarc.warc.WarcRecord property) is_http (fastwarc.warc.WarcRecord property) is_http() (in module fastwarc.warc) is_http_parsed (fastwarc.warc.WarcRecord property) is_warc_10() (in module fastwarc.warc) is_warc_11() (in module fastwarc.warc) iterate_http_chunks() (in module resiliparse.parse.http)

L

last_child (resiliparse.parse.html.DOMNode attribute) last_element_child (resiliparse.parse.html.DOMNode attribute)

Lz4Reader (class in fastwarc.stream_io) Lz4Writer (class in fastwarc.stream_io)

M

map_encoding_to_html5() (in module resiliparse.parse.encoding) matches() (resiliparse.parse.html.DOMCollection method) (resiliparse.parse.html.DOMNode method) MatchFiles (class in resiliparse.beam.fileio) mem_guard() (in module resiliparse.process_guard) MemGuard (class in resiliparse.process_guard) MemoryLimitExceeded metadata (fastwarc.warc.WarcRecordType attribute) module fastwarc.stream_io fastwarc.warc resiliparse.beam.coders resiliparse.beam.elasticsearch resiliparse.beam.fileio resiliparse.beam.textio resiliparse.beam.warcio resiliparse.extract.html2text resiliparse.itertools resiliparse.parse.encoding resiliparse.parse.html resiliparse.parse.http resiliparse.parse.lang resiliparse.process_guard

N

next (resiliparse.parse.html.DOMNode attribute) next_element (resiliparse.parse.html.DOMNode attribute)

no_type (fastwarc.warc.WarcRecordType attribute) NodeType (class in resiliparse.parse.html) NOTATION (resiliparse.parse.html.NodeType attribute)

O

OFFSET fastwarc-extract command line option OUTDIR resiliparse-lang-create-dataset command line option

OUTFILE fastwarc-recompress command line option

P

parent (resiliparse.parse.html.DOMNode attribute) parse() (resiliparse.parse.html.HTMLTree class method) parse_from_bytes() (resiliparse.parse.html.HTMLTree class method) parse_http() (fastwarc.warc.WarcRecord method) parse_warc_headers() (fastwarc.warc.WarcRecord method) prev (resiliparse.parse.html.DOMNode attribute)

prev_element (resiliparse.parse.html.DOMNode attribute) PROCESSING_INSTRUCTION (resiliparse.parse.html.NodeType attribute) progress() (in module resiliparse.process_guard) (resiliparse.process_guard.TimeGuard method) progress_loop() (in module resiliparse.itertools) (in module resiliparse.process_guard)

Q

query_selector() (resiliparse.parse.html.DOMCollection method) (resiliparse.parse.html.DOMNode method)

query_selector_all() (resiliparse.parse.html.DOMCollection method) (resiliparse.parse.html.DOMNode method)

R

read() (fastwarc.stream_io.WarcReader method) read_http_chunk() (in module resiliparse.parse.http) ReadAllFromText (class in resiliparse.beam.textio) ReadAllWarcs (class in resiliparse.beam.warcio) reader (fastwarc.warc.WarcRecord property) ReaderStaleError ReadFromText (class in resiliparse.beam.textio) readline() (fastwarc.warc.WarcRecordPayloadReader method) ReadWarcs (class in resiliparse.beam.warcio) record_date (fastwarc.warc.WarcRecord property) record_id (fastwarc.warc.WarcRecord property) record_type (fastwarc.warc.WarcRecord property) remove() (resiliparse.parse.html.DOMElementClassList method) remove_child() (resiliparse.parse.html.DOMNode method) replace_child() (resiliparse.parse.html.DOMNode method) request (fastwarc.warc.WarcRecordType attribute) reset() (resiliparse.parse.encoding.EncodingDetector method) resiliparse-html-benchmark command line option WARC_FILE resiliparse-lang-benchmark command line option --fasttext-model --rounds -f -r INFILE resiliparse-lang-create-dataset command line option --jobs --min-examples --test-size --val-size -j INDIR OUTDIR resiliparse-lang-download-wiki-dumps command line option --jobs --langs --outdir -j -l -o DUMPDATE resiliparse-lang-evaluate command line option --cutoff --fasttext-model --langs --print-cm --sort-lang --split --truncate -c -f -l -s -t INDIR

resiliparse-lang-train-vectors command line option --out-format --split --vector-size -f -l -s INDIR resiliparse.beam.coders module resiliparse.beam.elasticsearch module resiliparse.beam.fileio module resiliparse.beam.textio module resiliparse.beam.warcio module resiliparse.extract.html2text module resiliparse.itertools module resiliparse.parse.encoding module resiliparse.parse.html module resiliparse.parse.http module resiliparse.parse.lang module resiliparse.process_guard module ResiliparseGuardException resource (fastwarc.warc.WarcRecordType attribute) response (fastwarc.warc.WarcRecordType attribute) revisit (fastwarc.warc.WarcRecordType attribute)

S

seek() (fastwarc.stream_io.WarcReader method) set_bytes_content() (fastwarc.warc.WarcRecord method) set_bytes_payload() (fastwarc.warc.WarcRecord method) setattr() (resiliparse.parse.html.DOMNode method)

signal (resiliparse.process_guard.InterruptType attribute) stream_pos (fastwarc.warc.WarcRecord property) StreamError StrUtf8Coder (class in resiliparse.beam.coders) supported_langs() (in module resiliparse.parse.lang)

T

tag (resiliparse.parse.html.DOMNode attribute) tell() (fastwarc.stream_io.WarcReader method) text (resiliparse.parse.html.DOMNode attribute) TEXT (resiliparse.parse.html.NodeType attribute) time_guard() (in module resiliparse.process_guard)

TimeGuard (class in resiliparse.process_guard) title (resiliparse.parse.html.HTMLTree attribute) train_language_examples() (in module resiliparse.parse.lang) traverse_dom() (in module resiliparse.parse.html) type (resiliparse.parse.html.DOMNode attribute)

U

unknown (fastwarc.warc.WarcRecordType attribute)

update() (resiliparse.parse.encoding.EncodingDetector method) update_action() (in module resiliparse.beam.elasticsearch)

V

value (resiliparse.parse.html.DOMNode attribute)

verify_block_digest() (fastwarc.warc.WarcRecord method) verify_payload_digest() (fastwarc.warc.WarcRecord method)

W

WARC_BLOCK_DIGEST (fastwarc.warc.WarcHeader attribute) WARC_CONCURRENT_TO (fastwarc.warc.WarcHeader attribute) WARC_DATE (fastwarc.warc.WarcHeader attribute) WARC_FILE resiliparse-html-benchmark command line option WARC_FILENAME (fastwarc.warc.WarcHeader attribute) WARC_IDENTIFIED_PAYLOAD_TYPE (fastwarc.warc.WarcHeader attribute) WARC_IP_ADDRESS (fastwarc.warc.WarcHeader attribute) WARC_PAYLOAD_DIGEST (fastwarc.warc.WarcHeader attribute) WARC_PROFILE (fastwarc.warc.WarcHeader attribute) WARC_RECORD_ID (fastwarc.warc.WarcHeader attribute) WARC_REFERS_TO (fastwarc.warc.WarcHeader attribute) WARC_REFERS_TO_DATE (fastwarc.warc.WarcHeader attribute) WARC_REFERS_TO_TARGET_URI (fastwarc.warc.WarcHeader attribute) warc_retry() (in module resiliparse.itertools) WARC_SEGMENT_NUMBER (fastwarc.warc.WarcHeader attribute)

WARC_SEGMENT_ORIGIN_ID (fastwarc.warc.WarcHeader attribute) WARC_SEGMENT_TOTAL_LENGTH (fastwarc.warc.WarcHeader attribute) WARC_TARGET_URI (fastwarc.warc.WarcHeader attribute) WARC_TRUNCATED (fastwarc.warc.WarcHeader attribute) WARC_TYPE (fastwarc.warc.WarcHeader attribute) WARC_WARCINFO_ID (fastwarc.warc.WarcHeader attribute) WarcHeader (class in fastwarc.warc) WarcHeaderMap (class in fastwarc.warc) warcinfo (fastwarc.warc.WarcRecordType attribute) WarcReader (class in fastwarc.stream_io) WarcRecord (class in fastwarc.warc) WarcRecordPayloadReader (class in fastwarc.warc) WarcRecordType (class in fastwarc.warc) WarcWriter (class in fastwarc.stream_io) write() (fastwarc.stream_io.WarcWriter method) (fastwarc.warc.WarcRecord method)

Z

zstd_train_dictionary_from_continuous() (in module fastwarc.stream_io) zstd_train_dictionary_from_files() (in module fastwarc.stream_io)

zstd_train_dictionary_from_samples() (in module fastwarc.stream_io) ZstdReader (class in fastwarc.stream_io) ZstdWriter (class in fastwarc.stream_io)