#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
charsim 記事の図と本文中の数値を再計算・再生成するスクリプト。
記事: content/scraps/202607_charsim.html / 202607_charsim.en.html
- 実定義 (basic / avg-length) を素朴実装し §3.4・反例の値を検証
- §6 の簡略モデル (closed form) で optlen 図、§6.4 で burst 図を再生成
- 出力 SVG: charsim_optlen.svg, charsim_burst.svg と各 .en.svg（このディレクトリ）
"""
import math
from collections import Counter
# ---------------------------------------------------------------------------
# 実定義（素朴実装）
# ---------------------------------------------------------------------------
START = object()
END = object()
def ngram_counts(x, n):
"""境界記号を1個ずつ補い、境界だけの n-gram を除いた次数 n の多重集合."""
s = (START, *x, END)
c = Counter()
for i in range(len(s) - n + 1):
g = s[i:i + n]
if all(v is START or v is END for v in g):
continue
c[g] += 1
return c
def Lxn(x, n):
return sum(ngram_counts(x, n).values())
def overlap(t, r, n):
ct, cr = ngram_counts(t, n), ngram_counts(r, n)
return sum(min(ct[g], cr[g]) for g in ct.keys() & cr.keys())
def charsim(t, refs, mode, cap=32):
if not refs:
raise ValueError("at least one reference is required")
if cap < 1:
raise ValueError("cap must be positive")
Nprime = max([len(t)] + [len(r) for r in refs]) + 2
N = min(Nprime, cap)
Lt = sum(Lxn(t, n) / n for n in range(1, N + 1))
if mode == "basic":
ss = []
for r in refs:
Mi = sum(overlap(t, r, n) / n for n in range(1, N + 1))
Li = sum(Lxn(r, n) / n for n in range(1, N + 1))
denom = max(Lt, Li)
ss.append(Mi / denom if denom > 0 else 1.0)
return sum(ss) / len(ss)
else: # avg-length（主定義）
Mbar = sum((sum(overlap(t, r, n) for r in refs) / len(refs)) / n
for n in range(1, N + 1))
Lrbar = sum((sum(Lxn(r, n) for r in refs) / len(refs)) / n
for n in range(1, N + 1))
denom = max(Lt, Lrbar)
return Mbar / denom if denom > 0 else 1.0
# ---------------------------------------------------------------------------
# 検証：§3.4 と §3.3 反例
# ---------------------------------------------------------------------------
def verify():
print("== §3.4 単一参照 R=[cat] ==")
for t in ["cat", "cats", "ca", "catcat"]:
v = charsim(t, ["cat"], "avg")
print(f" charsim({t!r}; [cat]) = {v:.3f}")
print("== §3.3 反例 t='a', R=['a','abcdefghij'] ==")
print(f" basic = {charsim('a', ['a','abcdefghij'], 'basic'):.3f}")
print(f" avg = {charsim('a', ['a','abcdefghij'], 'avg'):.3f}")
# ---------------------------------------------------------------------------
# §6 簡略モデル（closed form, \tilde L_x = |x|, 当たる率 p 一定）
# charsim = p * min/max
# chrF = F_beta(P,R), P=p*min/lhat, R=p*min/l (beta=2)
# BLEU = P * BP, BP=1 (lhat>=l) else exp(1-l/lhat)
# ---------------------------------------------------------------------------
def m_optlen(lhat, l, p):
mn = p * min(lhat, l)
P = mn / lhat if lhat > 0 else 0.0
R = mn / l
cs = p * (min(lhat, l) / max(lhat, l)) if lhat > 0 else 0.0
F = (5 * P * R / (4 * P + R)) if (4 * P + R) > 0 else 0.0 # beta=2
bp = 1.0 if lhat >= l else (math.exp(1 - l / lhat) if lhat > 0 else 0.0)
bleu = P * bp
return cs, bleu, F
def centroid(metric_idx, l, p, lo=50, hi=150):
xs = list(range(lo, hi + 1))
w = [m_optlen(x, l, p)[metric_idx] for x in xs]
return sum(x * wi for x, wi in zip(xs, w)) / sum(w)
# ---------------------------------------------------------------------------
# §6.4 簡略モデル：分散した一致と連続ブロックのスコア比
# ---------------------------------------------------------------------------
def burst_ratio(N, l, q, weighted):
b = q * l
num = den = 0.0
for n in range(1, N + 1):
wn = (1.0 / n) if weighted else 1.0
m_block = max(b - n + 1, 0)
m_diff = (q ** n) * l
num += wn * m_block
den += wn * m_diff
return num / den
# ---------------------------------------------------------------------------
# SVG ヘルパ
# ---------------------------------------------------------------------------
def poly(points, color, w=2.4, dash=None):
d = f' stroke-dasharray="{dash}"' if dash else ""
pts = " ".join(f"{x:.1f},{y:.1f}" for x, y in points)
return f''
# ---------------------------------------------------------------------------
# optlen 図
# ---------------------------------------------------------------------------
def gen_optlen(path, l=100, p=0.8, lang="ja"):
labels = {
"ja": ("候補長 ", "（文字数）", "スコア", "対称な補助線"),
"en": ("Candidate length ", " (characters)", "Score", "Symmetric guide"),
}[lang]
X0, X1, Y0, Y1 = 72.0, 628.0, 344.0, 44.0 # ℓ̂:0..200, score:0..1
xmax = 200.0
def sx(lh): return X0 + (lh / xmax) * (X1 - X0)
def sy(sc): return Y0 + sc * (Y1 - Y0)
lhats = list(range(201))
cs = [(sx(lh), sy(m_optlen(lh, l, p)[0])) for lh in lhats]
bl = [(sx(lh), sy(m_optlen(lh, l, p)[1])) for lh in lhats]
cf = [(sx(lh), sy(m_optlen(lh, l, p)[2])) for lh in lhats]
out = []
out.append('')
with open(path, "w") as f:
f.write("\n".join(out) + "\n")
# ---------------------------------------------------------------------------
# burst 図
# ---------------------------------------------------------------------------
def gen_burst(path, l=100, q=0.8, lang="ja"):
labels = {
"ja": {
"equal": "×1：同じスコア", "cap": "n-gramの次数上限 ",
"ratio": "スコア比（連続／分散）", "uniform_value": "一様重み：×",
"weighted_value": "重み：×", "weighted": "重み（charsim）",
"uniform": "一様重み",
},
"en": {
"equal": "×1: equal scores", "cap": "Maximum n-gram order ",
"ratio": "Score ratio (block / dispersed)", "uniform_value": "Uniform weights: ×",
"weighted_value": " weights: ×", "weighted": " weights (charsim)",
"uniform": "Uniform weights",
},
}[lang]
X0, X1, Y0 = 72.0, 628.0, 344.0 # N:1..100, ratio:×1(=344)..×9(=44)
def sx(N): return X0 + (N - 1) / 99.0 * (X1 - X0)
def sy(r): return Y0 - (r - 1) / 8.0 * 300.0
Ns = list(range(1, 101))
uni = [(sx(N), sy(burst_ratio(N, l, q, False))) for N in Ns]
cha = [(sx(N), sy(burst_ratio(N, l, q, True))) for N in Ns]
r_uni = burst_ratio(100, l, q, False)
r_cha = burst_ratio(100, l, q, True)
out = []
out.append('')
with open(path, "w") as f:
f.write("\n".join(out) + "\n")
if __name__ == "__main__":
verify()
print("== §6.3 optlen の代表値 ==")
for lh in [20, 30, 70, 130]:
cs, bl, cf = m_optlen(lh, 100, 0.8)
print(f" λ̂={lh}: charsim={cs:.3f} BLEU={bl:.3f} chrF={cf:.3f}")
print(" 重心 chrF=%.1f BLEU=%.1f charsim=%.1f" %
(centroid(2, 100, 0.8), centroid(1, 100, 0.8), centroid(0, 100, 0.8)))
print("== §6.4 連続一致／分散した一致のスコア比 ==")
for n in [32, 100]:
print(f" 一様 N={n}: {burst_ratio(n,100,0.8,False):.1f}")
print(f" 1/n重み λ=100 (N={n}): {burst_ratio(n,100,0.8,True):.2f}")
for L in [100, 1000, 10000]:
print(f" 1/n重み λ={L} (N=λ): {burst_ratio(L,L,0.8,True):.1f}")
import os
# 日英両版の図を、このスクリプトと同じassets/配下に出力する。
here = os.path.dirname(os.path.abspath(__file__))
out = here
for lang in ("ja", "en"):
suffix = "" if lang == "ja" else ".en"
gen_optlen(os.path.join(out, f"charsim_optlen{suffix}.svg"), lang=lang)
gen_burst(os.path.join(out, f"charsim_burst{suffix}.svg"), lang=lang)
print("wrote", out + "/charsim_{optlen,burst}{,.en}.svg")