Skip to content

Multiple ranking functions #202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
apohllo wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Multiple ranking functions #202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d987da
new data and mapping
djstrong Apr 17, 2023
282faaf
Multiple sorting conditions
apohllo Apr 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 25 additions & 6 deletions research/elasticsearch/populate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,17 @@ def initialize_index(es: Elasticsearch):
"data.names.tokenized_name": {"type": "text", "similarity": "BM25_b0"},
"data.collection_description": {"type": "text", "similarity": "BM25"},
"data.collection_keywords": {"type": "text", "similarity": "BM25"},
# "template.collection_articles": {"type": "text", "similarity": "BM25"}, # TODO remove?
"template.collection_rank": {"type": "rank_feature"},
"metadata.members_count": {"type": "rank_feature"},
"template.members_rank_mean": {"type": "rank_feature"},
"template.members_rank_median": {"type": "rank_feature"},
"template.members_system_interesting_score_mean": {"type": "rank_feature"},
"template.members_system_interesting_score_median": {"type": "rank_feature"},
"template.valid_members_count": {"type": "rank_feature"},
"template.invalid_members_count": {"type": "rank_feature", "positive_score_impact": False},
"template.valid_members_ratio": {"type": "rank_feature"},
"template.nonavailable_members_count": {"type": "rank_feature"},
"template.nonavailable_members_ratio": {"type": "rank_feature"},
}
},
}
Expand All @@ -136,7 +144,7 @@ def insert_collection(es: Elasticsearch, collection: dict):


def insert_collections(es: Elasticsearch, collections: Iterable[dict]):
number_of_docs = 500000
number_of_docs = 561000
progress = tqdm(unit="docs", total=number_of_docs)
successes = 0

Expand All @@ -154,21 +162,28 @@ def insert_collections(es: Elasticsearch, collections: Iterable[dict]):

def gen(path, limit):
with jsonlines.open(path, 'r') as reader:
too_long = 0
for doc in islice(reader.iter(skip_empty=True, skip_invalid=True), limit):
# rank_feature must be positive

if doc['data']['collection_name'].startswith('Lists of'):
continue
doc['template']['collection_rank'] = max(1, doc['template']['collection_rank']) # remove?
doc['metadata']['members_count'] = len(doc['data']['names'])
# if doc['data']['collection_name'].startswith('Lists of'):
# continue
# doc['template']['collection_rank'] = max(1, doc['template']['collection_rank']) # remove?
# doc['metadata']['members_count'] = len(doc['data']['names'])

doc['template']['nonavailable_members_count'] += 1 #TODO?
doc['template']['invalid_members_count'] += 1 #TODO?

if doc['metadata']['members_count'] > 10000:
too_long += 1
continue

yield {
"_index": INDEX_NAME,
# "_type": '_doc',
"_source": doc
}
print(f'{too_long} collections too long')


if __name__ == '__main__':
Expand All @@ -188,9 +203,13 @@ def gen(path, limit):
else:
es = connect_to_elasticsearch(args.scheme, args.host, args.port, args.username, args.password)

es.indices.delete(index=INDEX_NAME)

initialize_index(es)

insert_collections(es, gen(args.input, args.limit))

search = es.search(index=INDEX_NAME, body={'query': {'bool': {}}})
print(f'Documents overall in {INDEX_NAME} - {len(search["hits"]["hits"])}')
# 103 collections too long
# Indexed 424307 documents
Loading