1
0
Fork 0

Finish implementing WaniKani sync

main
Marissa 5 years ago
parent 545fa265ed
commit 9d9e0e3746
  1. 1
      .gitignore
  2. 224
      addon/__init__.py
  3. 6
      addon/config.json

1
.gitignore vendored

@ -1,2 +1,3 @@
venv/ venv/
.idea/ .idea/
addon/meta.json

@ -1,12 +1,20 @@
import os.path import os.path
import re import re
import json import json
import unicodedata
import urllib.request import urllib.request
from anki import find
from anki.find import Finder
from anki.hooks import addHook
from aqt import mw from aqt import mw
from anki.sound import allSounds from anki.sound import allSounds
from anki.utils import stripHTMLMedia from anki.utils import stripHTMLMedia, splitFields, ids2str
from aqt.qt import QAction from aqt.qt import QAction
from collections import namedtuple from aqt.utils import showInfo, showText
from urllib.error import HTTPError
from functools import lru_cache
from time import time
def reshow_card(): def reshow_card():
@ -132,42 +140,67 @@ def wrap_list_and_add(owner, name, items):
def new_func(*args, **kwargs): def new_func(*args, **kwargs):
return old(*args, **kwargs) + items return old(*args, **kwargs) + items
setattr(owner, name, new_func) setattr(owner, name, new_func)
class Config(object): class Config(object):
__slots__ = ["wk_api_key", "kanji_query", "kanji_list_cache"] __slots__ = ["should_searching_be_fast", "wk_api_key", "kanji_global_query", "kanji_individual_query",
"kanji_list_cache"]
@classmethod @classmethod
def from_config(cls): def from_config(cls):
return cls.from_json(mw.addonManager.getConfig(__name__)) return cls.from_json(mw.addonManager.getConfig(__name__))
@classmethod @classmethod
def from_json(cls, json_obj): def from_json(cls, json_obj):
return Config(wk_api_key=json_obj.get("wk_api_key", ""), return Config(should_searching_be_fast=json_obj.get("should_searching_be_fast", False),
kanji_query=json_obj.get("kanji_query", ""), wk_api_key=json_obj.get("wk_api_key", ""),
kanji_global_query=json_obj.get("kanji_global_query", ""),
kanji_individual_query=json_obj.get("kanji_individual_query", ""),
kanji_list_cache=KanjiListCache.from_json(json_obj.get("kanji_list_cache", None))) kanji_list_cache=KanjiListCache.from_json(json_obj.get("kanji_list_cache", None)))
def __init__(self, wk_api_key="", kanji_query="", kanji_list_cache=None): def __init__(self, should_searching_be_fast=True, wk_api_key="", kanji_global_query="", kanji_individual_query="",
kanji_list_cache=None):
self.should_searching_be_fast = should_searching_be_fast
self.wk_api_key = wk_api_key self.wk_api_key = wk_api_key
self.kanji_query = kanji_query self.kanji_global_query = kanji_global_query
self.kanji_individual_query = kanji_individual_query
self.kanji_list_cache = kanji_list_cache or KanjiListCache() self.kanji_list_cache = kanji_list_cache or KanjiListCache()
def save(self): def save(self):
mw.addonManager.writeConfig(__name__, { mw.addonManager.writeConfig(__name__, {
"should_searching_be_fast": self.should_searching_be_fast,
"wk_api_key": self.wk_api_key, "wk_api_key": self.wk_api_key,
"kanji_query": self.kanji_query, "kanji_global_query": self.kanji_global_query,
"kanji_individual_query": self.kanji_individual_query,
"kanji_list_cache": self.kanji_list_cache.to_json(), "kanji_list_cache": self.kanji_list_cache.to_json(),
}) })
def kanji_query(self):
return self.kanji_global_query.format(
kanji=" or ".join(
self.kanji_individual_query.format(kanji=kanji)
for kanji in self.kanji_list_cache.definitions.values()))
class KanjiListCache(object): class KanjiListCache(object):
__slots__ = ["definitions", "last_list_etag", "last_list_update", "last_definition_etag", "last_definition_update"] __slots__ = ["definitions", "last_list_etag", "last_list_update", "last_definition_etag", "last_definition_update"]
@classmethod @classmethod
def from_json(cls, json_obj=None): def from_json(cls, json_obj=None):
if not json_obj: if not json_obj:
return return
return KanjiListCache(
definitions=json_obj.get("definitions", None),
last_list_etag=json_obj.get("last_list_etag", None),
last_list_update=json_obj.get("last_list_update", None),
last_definition_etag=json_obj.get("last_definition_etag", None),
last_definition_update=json_obj.get("last_definition_update", None)
)
def __init__(self, definitions=None, last_list_etag="", last_list_update="", last_definition_etag="", last_definition_update=""): def __init__(self, definitions=None, last_list_etag="", last_list_update="", last_definition_etag="",
last_definition_update=""):
self.definitions = dict(definitions or {}) self.definitions = dict(definitions or {})
self.last_list_etag = last_list_etag self.last_list_etag = last_list_etag
self.last_list_update = last_list_update self.last_list_update = last_list_update
@ -182,7 +215,33 @@ class KanjiListCache(object):
"last_list_etag": self.last_list_etag, "last_list_etag": self.last_list_etag,
"last_list_update": self.last_list_update} "last_list_update": self.last_list_update}
def update_unlocked_kanji(config): def update_unlocked_kanji(config):
list_etag, list_updated, updated_ids = get_updated_kanji_assignments(config)
previous_ids = set(config.kanji_list_cache.definitions.keys())
all_ids = updated_ids | previous_ids
defs_updated, defs_etag, defs_updates = get_kanji_definitions(config, all_ids, updated=UPDATED_KANJI)
missing_ids = updated_ids - set(updated_def_id for updated_def_id in defs_updates.keys())
if missing_ids:
_, _, defs_new = get_kanji_definitions(config, missing_ids, updated=NEW_KANJI)
else:
defs_new = ()
if list_updated:
config.kanji_list_cache.last_list_update = list_updated
if list_etag:
config.kanji_list_cache.last_list_etag = list_etag
if defs_updated:
config.kanji_list_cache.last_definition_update = defs_updated
if defs_etag:
config.kanji_list_cache.last_definition_etag = defs_etag
if defs_updates:
config.kanji_list_cache.definitions.update(defs_updates)
if defs_new:
config.kanji_list_cache.definitions.update(defs_new)
config.save()
def get_updated_kanji_assignments(config):
url = ( url = (
r"https://api.wanikani.com/v2/assignments" r"https://api.wanikani.com/v2/assignments"
r"?started=true" r"?started=true"
@ -191,45 +250,42 @@ def update_unlocked_kanji(config):
url += r"&updated_after=" + config.kanji_list_cache.last_list_update url += r"&updated_after=" + config.kanji_list_cache.last_list_update
list_etag = None list_etag = None
list_updated = None list_updated = None
ids = set() updated_ids = set()
while url: while url:
list_request = urllib.request.Request(url) list_request = urllib.request.Request(url)
list_request.add_header("Authorization", "Bearer {wk_api_key}".format(wk_api_key=config.wk_api_key)) list_request.add_header("Authorization", "Bearer {wk_api_key}".format(wk_api_key=config.wk_api_key))
list_request.add_header("Wanikani-Revision", "20170710") list_request.add_header("Wanikani-Revision", "20170710")
if config.kanji_list_cache.last_list_etag: if config.kanji_list_cache.last_list_etag:
list_request.add_header("If-None-Match", config.kanji_list_cache.last_list_etag) list_request.add_header("If-None-Match", config.kanji_list_cache.last_list_etag)
try:
with urllib.request.urlopen(list_request) as result: with urllib.request.urlopen(list_request) as result:
code = result.getcode()
headers = result.info() headers = result.info()
page = json.loads(result.read()) page = json.loads(result.read())
if code != 200: except HTTPError as ex:
raise ValueError("Got %s from list request %s: %s" % (code, url, json.dumps(page))) if ex.code == 304:
return None, None, set()
raise
list_etag = headers["ETag"] list_etag = headers["ETag"]
list_updated = page["data_updated_at"] list_updated = page["data_updated_at"]
ids.add(item["id"] for item in page["data"]) updated_ids.update(str(item["data"]["subject_id"]) for item in page["data"])
if page["pages"]["next_page"]: if page["pages"].get("next_url", None):
url = page["pages"]["next_page"] url = page["pages"]["next_url"]
else: else:
url = None url = None
previous_ids = set(int(cached_id) for cached_id in config.kanji_list_cache.definitions.keys()) return list_etag, list_updated, updated_ids
previous_kanji = set(config.kanji_list_cache.definitions.values())
all_ids = ids | previous_ids
defs_updated, defs_etag, defs_updates = get_updated_kanji_definitions(config, all_ids)
config.kanji_list_cache.definitions.update(defs_updates)
missing_ids = set(config.kanji_list_cache.definitions.keys()) - previous_ids
# TODO:
# Retrieve any items corresponding to ids which are not present in the cache without any cache specifiers
# Update the four cache specifiers
# Save the configuration
def get_updated_kanji_definitions(config, all_ids): NEW_KANJI = "New"
UPDATED_KANJI = "Updated"
def get_kanji_definitions(config, kanji_ids, updated=UPDATED_KANJI):
url = ( url = (
r"https://api.wanikani.com/v2/subjects" r"https://api.wanikani.com/v2/subjects"
r"?types=kanji" r"?types=kanji"
r"&ids=%s" % (",".join(all_ids),) r"&ids=%s" % (",".join(kanji_ids),)
) )
if config.kanji_list_cache.last_definition_update: if updated != NEW_KANJI and config.kanji_list_cache.last_definition_update:
url += r"&updated_after=" + config.kanji_list_cache.last_definition_update url += r"&updated_after=" + config.kanji_list_cache.last_definition_update
etag = None etag = None
updated = None updated = None
@ -238,31 +294,77 @@ def get_updated_kanji_definitions(config, all_ids):
list_request = urllib.request.Request(url) list_request = urllib.request.Request(url)
list_request.add_header("Authorization", "Bearer {wk_api_key}".format(wk_api_key=config.wk_api_key)) list_request.add_header("Authorization", "Bearer {wk_api_key}".format(wk_api_key=config.wk_api_key))
list_request.add_header("Wanikani-Revision", "20170710") list_request.add_header("Wanikani-Revision", "20170710")
if config.kanji_list_cache.last_definition_etag: if updated != NEW_KANJI and config.kanji_list_cache.last_definition_etag:
list_request.add_header("If-None-Match", config.kanji_list_cache.last_definition_etag) list_request.add_header("If-None-Match", config.kanji_list_cache.last_definition_etag)
try:
with urllib.request.urlopen(list_request) as result: with urllib.request.urlopen(list_request) as result:
code = result.getcode()
headers = result.info() headers = result.info()
page = json.loads(result.read()) page = json.loads(result.read())
if code != 200: except HTTPError as ex:
raise ValueError("Got %s from definition request %s: %s" % (code, url, json.dumps(page))) if ex.code == 304:
return None, None, {}
raise
etag = headers["ETag"] etag = headers["ETag"]
updated = page["data_updated_at"] updated = page["data_updated_at"]
for item in page["data"]: for item in page["data"]:
updated_definitions[item["id"]] = item["characters"] item_id = str(item["id"])
if page["pages"]["next_page"]: characters = item["data"]["characters"]
url = page["pages"]["next_page"] updated_definitions[item_id] = characters
if page["pages"].get("next_url", None):
url = page["pages"]["next_url"]
else: else:
url = None url = None
return updated, etag, updated_definitions return updated, etag, updated_definitions
def get_new_kanji_definitions(config, new_ids): def find_unsuspendable_kanji_cards(config):
pass return set(mw.col.findCards(query))
def sync_wani_kani(): def sync_wani_kani():
pass unlocked_kanji_search_time = time()
config = Config.from_config()
update_unlocked_kanji(config=config)
unlocked_kanji_search_time = time() - unlocked_kanji_search_time
card_search_time = time()
query = config.kanji_query()
kanji_to_unsuspend = set(mw.col.findCards(query))
card_search_time = time() - card_search_time
if not kanji_to_unsuspend:
showText(
txt=(
"Retrieved {unlocked_kanji} unlocked kanji from WaniKani in {unlocked_kanji_search_time:f} seconds.\n\n"
"No cards to unsuspend in {card_search_time:f} seconds when searching {speed}.\n\nUsed query:\n{query}"
).format(
unlocked_kanji=len(config.kanji_list_cache.definitions),
unlocked_kanji_search_time=unlocked_kanji_search_time,
card_search_time=card_search_time,
speed=("quickly" if config.should_searching_be_fast else "slowly"),
query=query),
copyBtn=True)
return
mw.col.sched.unsuspendCards(kanji_to_unsuspend)
mw.reset()
card_text = []
for card_id in kanji_to_unsuspend:
card = mw.col.getCard(card_id)
note = card.note()
card_text.append("cid#{id} {kanji} (Lv. {level}, {onyomi} / {kunyomi}, {meaning})".format(
id=note.id,
kanji=note["Kanji"],
level=note["level"], onyomi=note["ONyomi"], kunyomi=note["KUNyomi"], meaning=note["Meaning"]))
showText(
txt=("Retrieved {unlocked_kanji} unlocked kanji from WaniKani in {unlocked_kanji_search_time:f} seconds.\n\n"
"Found {unsuspend_cards} card(s) to unsuspend in {card_search_time:f} seconds "
"when searching {speed}.\n\nUsed query:\n{query}\n\nUnsuspended cards:\n{card_list}").format(
unlocked_kanji=len(config.kanji_list_cache.definitions),
unlocked_kanji_search_time=unlocked_kanji_search_time,
unsuspend_cards=len(kanji_to_unsuspend),
card_search_time=card_search_time,
speed=("quickly" if config.should_searching_be_fast else "slowly"),
query=query,
card_list="\n".join(card_text)),
copyBtn=True)
new_shortcuts = ( new_shortcuts = (
@ -270,7 +372,6 @@ new_shortcuts = (
["Shift+m", swap_meaning_and_extra_info], ["Shift+m", swap_meaning_and_extra_info],
) )
new_context_menu_items = ( new_context_menu_items = (
None, None,
["Toggle Sound", "Shift+S", toggle_sound], ["Toggle Sound", "Shift+S", toggle_sound],
@ -284,3 +385,46 @@ wrap_list_and_add(mw.reviewer, "_contextMenu", new_context_menu_items)
action = QAction("Load WaniKani data", mw) action = QAction("Load WaniKani data", mw)
action.triggered.connect(sync_wani_kani) action.triggered.connect(sync_wani_kani)
mw.form.menuTools.addAction(action) mw.form.menuTools.addAction(action)
@lru_cache(maxsize=None)
def normalize_field_name(name):
return unicodedata.normalize("NFC", name.lower())
_findFieldButSlowly = find.Finder._findField
def _findFieldButFaster(field, val):
field = normalize_field_name(field)
val = val.replace("*", "%")
models_with_field = [
str(m['id']) for m in mw.col.models.all()
if any(normalize_field_name(f['name']) == field for f in m['flds'])]
return "n.mid in %s and field_by_model_id_and_name(n.mid, n.flds, '%s') like '%s' escape '\\'" % (
ids2str(models_with_field), field.replace("'", "''"), val.replace("'", "''"))
class FinderButFast(find.Finder):
def __init__(self, *args, **kwargs):
if Config.from_config().should_searching_be_fast:
self._findField = _findFieldButFaster
super(FinderButFast, self).__init__(*args, **kwargs)
def make_searching_fast():
models = mw.col.models
def get_field_by_model_id_and_name(id, flds, name):
m = models.get(id)
if not m:
return
for f in m['flds']:
if normalize_field_name(f['name']) == name:
return splitFields(flds)[f['ord']]
mw.col.db._db.create_function("field_by_model_id_and_name", 3, get_field_by_model_id_and_name)
addHook("profileLoaded", make_searching_fast)
find.Finder = FinderButFast

@ -1,5 +1,7 @@
{ {
"should_searching_be_fast": true,
"wk_api_key": "", "wk_api_key": "",
"kanji_query": "tag:kanji tag:writing is:suspended id:k_{kanji}", "kanji_global_query": "\"deck:\u66f8\u304f\u7df4\u7fd2\" \"note:Writing Kanji\" is:suspended ({kanji})",
"kanji_list_cache": "" "kanji_individual_query": "id:k_{kanji}",
"kanji_list_cache": null
} }
Loading…
Cancel
Save