CJK features for ZH,JA,KO wiki

.gitignore

@@ -13,6 +13,9 @@ ipython/.ipynb_checkpoints/
 # Temporary text editor files
 *~
 .*.sw?
+.ipynb_checkpoints/
+.vscode/
+*.ipynb
 
 # C extensions
 *.so

Makefile

@@ -46,7 +46,7 @@ models: \
 		srwiki_models \
 		svwiki_models \
 		tawiki_models \
-		translatewiki_models \
+#		translatewiki_models \
 		trwiki_models \
 		ukwiki_models \
 		urwiki_models \
@@ -96,7 +96,7 @@ tuning_reports: \
 		srwiki_tuning_reports \
 		svwiki_tuning_reports \
 		tawiki_tuning_reports \
-		translatewiki_tuning_reports \
+#		translatewiki_tuning_reports \
 		trwiki_tuning_reports \
 		ukwiki_tuning_reports \
 		urwiki_tuning_reports \
@@ -2448,7 +2448,7 @@ models/jawiki.goodfaith.gradient_boosting.model: \
 		-p 'learning_rate=0.1' \
 		-p 'max_depth=1' \
 		-p 'max_features="log2"' \
-		-p 'min_samples_leaf=7' \
+		-p 'min_samples_leaf=3' \
 		-p 'n_estimators=300' \
 		--label-weight $(goodfaith_weight) \
 		--pop-rate "true=0.00461076560154407" \
@@ -2522,10 +2522,11 @@ models/kowiki.damaging.gradient_boosting.model: \
 		editquality.feature_lists.kowiki.damaging \
 		damaging \
 		--version=$(damaging_major_minor).0 \
-		-p 'learning_rate=0.01' \
-		-p 'max_depth=7' \
+		-p 'learning_rate=0.5' \
+		-p 'max_depth=1' \
 		-p 'max_features="log2"' \
-		-p 'n_estimators=700' \
+		-p 'min_samples_leaf=5' \
+		-p 'n_estimators=500' \
 		--label-weight $(damaging_weight) \
 		--pop-rate "true=0.038665452792802445" \
 		--pop-rate "false=0.9613345472071976" \
@@ -2559,7 +2560,8 @@ models/kowiki.goodfaith.gradient_boosting.model: \
 		-p 'learning_rate=0.01' \
 		-p 'max_depth=7' \
 		-p 'max_features="log2"' \
-		-p 'n_estimators=700' \
+		-p 'min_samples_leaf=13' \
+		-p 'n_estimators=500' \
 		--label-weight $(goodfaith_weight) \
 		--pop-rate "true=0.9807743801210304" \
 		--pop-rate "false=0.019225619878969646" \
@@ -4063,11 +4065,11 @@ models/zhwiki.damaging.gradient_boosting.model: \
 		editquality.feature_lists.zhwiki.damaging \
 		damaging \
 		--version=$(damaging_major_minor).0 \
-		-p 'learning_rate=0.01' \
-		-p 'max_depth=3' \
+		-p 'learning_rate=0.1' \
+		-p 'max_depth=1' \
 		-p 'max_features="log2"' \
-		-p 'min_samples_leaf=7' \
-		-p 'n_estimators=700' \
+		-p 'min_samples_leaf=3' \
+		-p 'n_estimators=300' \
 		--label-weight $(damaging_weight) \
 		--pop-rate "true=0.0405" \
 		--pop-rate "false=0.9595" \
@@ -4098,11 +4100,11 @@ models/zhwiki.goodfaith.gradient_boosting.model: \
 		editquality.feature_lists.zhwiki.goodfaith \
 		goodfaith \
 		--version=$(goodfaith_major_minor).0 \
-		-p 'learning_rate=0.01' \
-		-p 'max_depth=3' \
+		-p 'learning_rate=0.1' \
+		-p 'max_depth=1' \
 		-p 'max_features="log2"' \
-		-p 'min_samples_leaf=5' \
-		-p 'n_estimators=500' \
+		-p 'min_samples_leaf=3' \
+		-p 'n_estimators=300' \
 		--label-weight $(goodfaith_weight) \
 		--pop-rate "true=0.9682" \
 		--pop-rate "false=0.03180000000000005" \

config/wikis/jawiki.yaml

@@ -62,5 +62,5 @@ models:
         max_depth: 1
         max_features: log2
         n_estimators: 300
-        min_samples_leaf: 7
+        min_samples_leaf: 3
 

config/wikis/kowiki.yaml

@@ -45,10 +45,11 @@ models:
     cv_train:
       algorithm: GradientBoosting
       parameters:
-        learning_rate: 0.01
-        max_depth: 7
+        learning_rate: 0.5
+        max_depth: 1
         max_features: log2
-        n_estimators: 700
+        n_estimators: 500
+        min_samples_leaf: 5
   goodfaith:
     observations: labeled_revisions.w_cache.20k_2016
     label: goodfaith
@@ -60,4 +61,5 @@ models:
         learning_rate: 0.01
         max_depth: 7
         max_features: log2
-        n_estimators: 700
+        n_estimators: 500
+        min_samples_leaf: 13

config/wikis/zhwiki.yaml

@@ -43,11 +43,11 @@ models:
     cv_train:
       algorithm: GradientBoosting
       parameters:
-        learning_rate: 0.01
-        max_depth: 3
+        learning_rate: 0.1
+        max_depth: 1
         max_features: log2
-        n_estimators: 700
-        min_samples_leaf: 7
+        n_estimators: 300
+        min_samples_leaf: 3
   goodfaith:
     observations: labeled_revisions.w_cache.100k_2016
     label: goodfaith
@@ -56,8 +56,8 @@ models:
     cv_train:
       algorithm: GradientBoosting
       parameters:
-        learning_rate: 0.01
-        max_depth: 3
+        learning_rate: 0.1
+        max_depth: 1
         max_features: log2
-        n_estimators: 500
-        min_samples_leaf: 5
+        n_estimators: 300
+        min_samples_leaf: 3

editquality/about.py

@@ -1,5 +1,5 @@
 __name__ = "editquality"
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 __author__ = "Aaron Halfaker"
 __author_email__ = "ahalfaker@wikimedia.org"
 __description__ = "A library for performing automatic detection of the " + \

editquality/feature_lists/jawiki.py

@@ -21,7 +21,7 @@
 ]
 
 damaging = wikipedia.page + \
-    wikitext.parent + wikitext.diff + mediawiki.user_rights + \
+    wikitext.parent_cjk + wikitext.diff_cjk + mediawiki.user_rights + \
     mediawiki.protected_user + mediawiki.comment + \
     badwords + informals + \
     enwiki.badwords + enwiki.informals

editquality/feature_lists/kowiki.py

@@ -21,10 +21,10 @@
 ]
 
 damaging = wikipedia.page + \
-    wikitext.parent + wikitext.diff + mediawiki.user_rights + \
+    wikitext.parent_cjk + wikitext.diff_cjk + mediawiki.user_rights + \
     mediawiki.protected_user + mediawiki.comment + \
     badwords + informals + \
-    enwiki.badwords + enwiki.informals + enwiki.dict_words
+    enwiki.badwords + enwiki.informals
 "Damaging Features"
 
 reverted = damaging

editquality/feature_lists/tests/test_cjk_wiki.py

@@ -0,0 +1,65 @@
+from revscoring.dependencies import solve
+from revscoring.datasources import revision_oriented
+from .. import zhwiki
+from .. import jawiki
+from .. import kowiki
+
+
+r_text = revision_oriented.revision.text
+p_text = revision_oriented.revision.parent.text
+
+
+def test_zhwiki():
+    p_text_text = """
+    中國是位於亞歐大陸的國家或地理區域。此名稱最早见于西周，
+    回頭見 拜拜 回見
+    """
+    r_text_text = """
+    中國中國是位於亞歐大陸的國家或地理區域。此名稱最早见于西周，
+    回頭見 拜拜 回見
+    """
+    cache = {p_text: p_text_text,
+             r_text: r_text_text}
+
+    assert (list(solve(zhwiki.wikitext.diff_cjk, cache=cache)) ==
+            [2, 1.0, 1.0])
+    assert ([round(i) for i in solve(zhwiki.wikitext
+                                           .parent_cjk, cache=cache)] ==
+            [4, 3, 2, 1])
+
+
+def test_jawiki():
+    p_text_text = """
+    敗戦後は桑原武夫の『第二芸術－現代俳句について』
+    （1946年）によって、短詩型である俳句の限界が指摘された。
+    """
+    r_text_text = """
+    敗戦後は桑原武夫の『第二芸術－現代俳句について』
+    （1946年）によって、短詩型である俳句の限界が指摘された た。
+    """
+    cache = {p_text: p_text_text,
+             r_text: r_text_text}
+    assert (list(solve(jawiki.wikitext.diff_cjk, cache=cache)) ==
+            [1, 2.0, 1.0])
+    assert ([round(i) for i in solve(jawiki.wikitext
+                                           .parent_cjk, cache=cache)] ==
+            [4, 3, 1, 1])
+
+
+def test_kowiki():
+    p_text_text = """
+    국어사 자료에서 ‘김치’가 소급하는 최초의 형태는
+     16세기의 ‘딤치’이다.
+    """
+    r_text_text = """
+    국어사 자료에서 ‘김치’가 소급하는 최초의 형태는
+     16세기의 ‘딤치’이다 다.
+    """
+    cache = {p_text: p_text_text,
+             r_text: r_text_text}
+
+    assert (list(solve(kowiki.wikitext.diff_cjk, cache=cache)) ==
+            [1, 2.0, 1.0])
+    assert ([round(i) for i in solve(kowiki.wikitext
+                                           .parent_cjk, cache=cache)] ==
+            [3, 3, 2, 1])

editquality/feature_lists/wikitext.py

@@ -33,6 +33,25 @@ def _process_new_longest(p_longest, r_longest):
         name="revision.parent.markups_per_token"),
 ]
 
+parent_cjk = [
+    log(wikitext.revision.parent.cjk_chars + 1),
+    log(wikitext.revision.parent.cjk.cjks + 1),
+    log(wikitext.revision.parent.headings + 1),
+    log(wikitext.revision.parent.wikilinks + 1),
+    log(wikitext.revision.parent.external_links + 1),
+    log(wikitext.revision.parent.templates + 1),
+    log(wikitext.revision.parent.ref_tags + 1),
+    div(wikitext.revision.parent.cjk_chars,
+        max(wikitext.revision.parent.cjk.cjks, 1),
+        name="revision.parent.cjk.cjkchars_per_cjkwordthing"),
+    div(wikitext.revision.parent.cjk.cjks,
+        max(wikitext.revision.parent.cjk.tokens, 1),
+        name="revision.parent.cjk.cjkwordthings_per_token"),
+    div(wikitext.revision.parent.markups,
+        max(wikitext.revision.parent.cjk.tokens, 1),
+        name="revision.parent.markups_per_token"),
+]
+
 diff = [
     wikitext.revision.diff.markup_delta_sum,
     wikitext.revision.diff.markup_delta_increase,
@@ -91,3 +110,56 @@ def _process_new_longest(p_longest, r_longest):
             depends_on=[wikitext.revision.parent.longest_repeated_char,
                         wikitext.revision.longest_repeated_char])
 ]
+
+diff_cjk = [
+    wikitext.revision.diff.markup_delta_sum,
+    wikitext.revision.diff.markup_delta_increase,
+    wikitext.revision.diff.markup_delta_decrease,
+    wikitext.revision.diff.markup_prop_delta_sum,
+    wikitext.revision.diff.markup_prop_delta_increase,
+    wikitext.revision.diff.markup_prop_delta_decrease,
+    wikitext.revision.diff.number_delta_sum,
+    wikitext.revision.diff.number_delta_increase,
+    wikitext.revision.diff.number_delta_decrease,
+    wikitext.revision.diff.number_prop_delta_sum,
+    wikitext.revision.diff.number_prop_delta_increase,
+    wikitext.revision.diff.number_prop_delta_decrease,   
+    sub(wikitext.revision.cjk_chars,
+        wikitext.revision.parent.cjk_chars,
+        name="revision.diff.cjkchars_change"),
+    sub(wikitext.revision.cjk.tokens,
+        wikitext.revision.parent.cjk.tokens,
+        name="revision.diff.cjk.tokens_change_inc_cjktokens"),
+    sub(wikitext.revision.cjk.cjks,
+        wikitext.revision.parent.cjk.cjks,
+        name="revision.diff.cjk.cjkwordthings_change"),
+    sub(wikitext.revision.markups,
+        wikitext.revision.parent.markups,
+        name="revision.diff.markups_change"),
+    sub(wikitext.revision.headings,
+        wikitext.revision.parent.headings,
+        name="revision.diff.headings_change"),
+    sub(wikitext.revision.external_links,
+        wikitext.revision.parent.external_links,
+        name="revision.diff.external_links_change"),
+    sub(wikitext.revision.wikilinks,
+        wikitext.revision.parent.wikilinks,
+        name="revision.diff.wikilinks_change"),
+    sub(wikitext.revision.templates,
+        wikitext.revision.parent.templates,
+        name="revision.diff.templates_change"),
+    sub(wikitext.revision.tags,
+        wikitext.revision.parent.tags,
+        name="revision.diff.tags_change"),
+    sub(wikitext.revision.ref_tags,
+        wikitext.revision.parent.ref_tags,
+        name="revision.diff.ref_tags_change"),
+    Feature("revision.diff.longest_new_token",
+            _process_new_longest, returns=int,
+            depends_on=[wikitext.revision.parent.longest_token,
+                        wikitext.revision.longest_token]),
+    Feature("revision.diff.longest_new_repeated_char",
+            _process_new_longest, returns=int,
+            depends_on=[wikitext.revision.parent.longest_repeated_char,
+                        wikitext.revision.longest_repeated_char])
+]

editquality/feature_lists/zhwiki.py

@@ -1,5 +1,4 @@
 from revscoring.languages import chinese
-
 from . import enwiki, mediawiki, wikipedia, wikitext
 
 badwords = [
@@ -28,8 +27,9 @@
     chinese.words_to_watch.revision.diff.match_prop_delta_increase,
     chinese.words_to_watch.revision.diff.match_prop_delta_decrease
 ]
+
 damaging = wikipedia.page + \
-    wikitext.parent + wikitext.diff + mediawiki.user_rights + \
+    wikitext.parent_cjk + wikitext.diff_cjk + mediawiki.user_rights + \
     mediawiki.protected_user + mediawiki.comment + \
     badwords + informals + words_to_watch + \
     enwiki.badwords + enwiki.informals