OSDN Git Service

a1a21fe4ead21da77edd3af909cc2d9ce113b2f6
[luatex-ja/luatexja.git] / src / luatexja-core.lua
1 local node_type = node.type
2 local has_attr = node.has_attribute
3 local node_insert_before = node.insert_before
4 local node_insert_after = node.insert_after
5 local node_hpack = node.hpack
6 local round = tex.round
7 local node_new = node.new
8 local id_glyph = node.id('glyph')
9 local id_glue_spec = node.id('glue_spec')
10 local id_glue = node.id('glue')
11 local id_whatsit = node.id('whatsit')
12 local next_node = node.next
13 local attr_jchar_class = luatexbase.attributes['luatexja@charclass']
14 local attr_curjfnt = luatexbase.attributes['luatexja@curjfnt']
15 local attr_yablshift = luatexbase.attributes['luatexja@yablshift']
16
17 -- error messages
18 function ltj.error(s,t)
19   tex.error('LuaTeX-ja error: ' .. s ,t) 
20 end
21
22 -- Three aux. functions, bollowed from tex.web
23 local unity=65536
24 local function print_scaled(s)
25    local out=''
26    local delta=10
27    if s<0 then 
28       out=out..'-'; s=-s
29    end
30    out=out..tostring(math.floor(s/unity)) .. '.'
31    s=10*(s%unity)+5
32    repeat
33       if delta>unity then s=s+32768-50000 end
34       out=out .. tostring(math.floor(s/unity)) 
35       s=10*(s%unity)
36       delta=delta*10
37    until s<=delta
38    return out
39 end
40
41 local function print_glue(d,order)
42    local out=print_scaled(d)
43    if order>0 then
44       out=out..'fi'
45       while order>1 do
46          out=out..'l'; order=order-1
47       end
48    else 
49       out=out..'pt'
50    end
51    return out
52 end
53
54 local function print_spec(p)
55    local out=print_scaled(p.width)..'pt'
56    if p.stretch~=0 then
57       out=out..' plus '..print_glue(p.stretch,p.stretch_order)
58    end
59    if p.shrink~=0 then
60       out=out..' minus '..print_glue(p.shrink,p.shrink_order)
61    end
62 return out
63 end
64
65 -- return true if and only if p is a Japanese character node
66 local function is_japanese_glyph_node(p)
67    return p and (p.id==id_glyph) 
68    and (p.font==has_attr(p,attr_curjfnt))
69 end
70
71 ---------- Stack table
72 ---- ltj.stack_ch_table [stack_level] : 情報を格納したテーブル
73 ----   .auto_spacing, .auto_xspacing: \autospacing etc.
74 ----   [chr_code].pre, [chr_code].post, [chr_code].xsp
75
76 ltj.stack_ch_table={}; ltj.stack_ch_table[0]={}
77
78 local function new_stack_level()
79   local i = tex.getcount('ltj@stack@pbp')
80   if tex.currentgrouplevel > tex.getcount('ltj@group@level@pbp') then
81     i = i+1 -- new stack level
82     tex.setcount('ltj@group@level@pbp', tex.currentgrouplevel)
83     for j,v in pairs(ltj.stack_ch_table) do -- clear the stack above i
84       if j>=i then ltj.stack_ch_table[j]=nil end
85     end
86     ltj.stack_ch_table[i] = table.fastcopy(ltj.stack_ch_table[i-1])
87     tex.setcount('ltj@stack@pbp', i)
88   end
89   return i
90 end
91 function ltj.set_ch_table(g,m,c,p)
92   local i = new_stack_level()
93   if not ltj.stack_ch_table[i][c] then ltj.stack_ch_table[i][c] = {} end
94   ltj.stack_ch_table[i][c][m] = p
95   if g=='global' then
96     for j,v in pairs(ltj.stack_ch_table) do 
97       if not ltj.stack_ch_table[j][c] then ltj.stack_ch_table[j][c] = {} end
98       ltj.stack_ch_table[j][c][m] = p
99     end
100   end
101 end
102
103 local function get_penalty_table(m,c)
104   local i = tex.getcount('ltj@stack@pbp')
105   i = ltj.stack_ch_table[i][c]
106   if i then i=i[m] end
107   return i or 0
108 end
109
110 local function get_inhibit_xsp_table(c)
111   local i = tex.getcount('ltj@stack@pbp')
112   i = ltj.stack_ch_table[i][c]
113   if i then i=i.xsp end
114   return i or 3
115 end
116
117 --------
118 function ltj.out_ja_parameter_one(k)
119    if k == 'yabaselineshift' then
120       tex.write(print_scaled(tex.getattribute('luatexja@yablshift'))..'pt')
121    elseif k == 'ykbaselineshift' then
122       tex.write(print_scaled(tex.getattribute('luatexja@ykblshift'))..'pt')
123    elseif k == 'kanjiskip' then
124       tex.write(print_spec(tex.getskip('kanjiskip')))
125    elseif k == 'xkanjiskip' then
126       tex.write(print_spec(tex.getskip('xkanjiskip')))
127    elseif k == 'jcharwidowpenalty' then
128       tex.write(tex.getcount('jcharwidowpenalty'))
129    elseif k == 'autospacing' then
130       tex.write(tostring(ltj.auto_spacing))
131    elseif k == 'autoxspacing' then
132       tex.write(tostring(ltj.auto_xspacing))
133    elseif k == 'differentjfm' then
134       if ltj.calc_between_two_jchar_aux==ltj.calc_between_two_jchar_aux_large then
135          tex.write('large')
136       elseif ltj.calc_between_two_jchar_aux==ltj.calc_between_two_jchar_aux_small then
137          tex.write('small')
138       elseif ltj.calc_between_two_jchar_aux==ltj.calc_between_two_jchar_aux_average then
139          tex.write('average')
140       elseif ltj.calc_between_two_jchar_aux==ltj.calc_between_two_jchar_aux_both then
141          tex.write('both')
142       else -- This can't happen.
143          tex.write('???')
144       end
145    end
146 end
147
148 function ltj.out_ja_parameter_two(k,c)
149    if k == 'prebreakpenalty' then
150       tex.write(get_penalty_table('pre',c))
151    elseif k == 'postbreakpenalty' then
152       tex.write(get_penalty_table('post',c))
153    elseif k == 'cjkxspmode' then
154       local i = get_inhibit_xsp_table(c)
155       if i==0 then tex.write('inhibit')
156       elseif i==1 then  tex.write('postonly')
157       elseif i==2 then  tex.write('preonly')
158       else tex.write('allow')
159       end
160    elseif k == 'asciixspmode' then
161       local i = get_inhibit_xsp_table(c)
162       if i==0 then tex.write('inhibit')
163       elseif i==2 then  tex.write('postonly')
164       elseif i==1 then  tex.write('preonly')
165       else tex.write('allow')
166       end
167    end
168 end
169
170
171 --------- 
172 function ltj.print_global()
173   if ltj.isglobal=='global' then tex.sprint('\\global') end
174 end
175
176 function ltj.create_ihb_node()
177    local g=node_new(id_whatsit, node.subtype('user_defined'))
178    g.user_id=30111; g.type=number; g.value=1
179    node.write(g)
180 end
181
182
183 local function find_size_metric(px)
184    if is_japanese_glyph_node(px) then
185       return ltj.font_metric_table[px.font].size, ltj.font_metric_table[px.font].jfm
186    else 
187       return nil, nil
188    end
189 end
190
191 local function new_jfm_glue(size,mt,bc,ac)
192 -- mt: metric key, bc, ac: char classes
193    local g=nil
194    local h
195    local w=bc*0x800+ac
196    if ltj.metrics[mt].glue[w] then
197       h=node_new(id_glue_spec)
198       h.width  =round(size*ltj.metrics[mt].glue[w].width)
199       h.stretch=round(size*ltj.metrics[mt].glue[w].stretch)
200       h.shrink =round(size*ltj.metrics[mt].glue[w].shrink)
201       h.stretch_order=0; h.shrink_order=0
202       g=node_new(id_glue)
203       g.subtype=0; g.spec=h; return g
204    elseif ltj.metrics[mt].kern[w] then
205       g=node_new(node.id('kern'))
206       g.subtype=0; g.kern=round(size*ltj.metrics[mt].kern[w]); return g
207    else
208       return nil
209    end
210 end
211
212
213 function calc_between_two_jchar(q,p)
214    -- q, p: node (possibly null)
215    local ps,pm,qs,qm,g,h
216    if not p then -- q is the last node
217       qs, qm = find_size_metric(q)
218       if not qm then 
219          return nil
220       else
221          g=new_jfm_glue(qs,qm,
222                             has_attr(q,attr_jchar_class),
223                                 ltj.find_char_type('boxbdd',qm))
224       end
225    elseif not q then
226       -- p is the first node etc.
227       ps, pm = find_size_metric(p)
228       if not pm then
229          return nil
230       else
231          g=new_jfm_glue(ps,pm,
232                                 ltj.find_char_type('boxbdd',pm),
233                                 has_attr(p,attr_jchar_class))
234       end
235    else -- p and q are not nil
236       qs, qm = find_size_metric(q)
237       ps, pm = find_size_metric(p)
238       if (not pm) and (not qm) then 
239          -- Both p and q are NOT Japanese glyph node
240          return nil
241       elseif (qs==ps) and (qm==pm) then 
242          -- Both p and q are Japanese glyph node, and same metric and size
243          g=new_jfm_glue(ps,pm,
244                             has_attr(q,attr_jchar_class),
245                             has_attr(p,attr_jchar_class))
246       elseif not qm then
247          -- q is not Japanese glyph node
248          g=new_jfm_glue(ps,pm,
249                             ltj.find_char_type('jcharbdd',pm),
250                             has_attr(p,attr_jchar_class))
251       elseif not pm then
252          -- p is not Japanese glyph node
253          g=new_jfm_glue(qs,qm,
254                             has_attr(q,attr_jchar_class),
255                             ltj.find_char_type('jcharbdd',qm))
256       else
257          g=new_jfm_glue(qs,qm,
258                             has_attr(q,attr_jchar_class),
259                             ltj.find_char_type('diffmet',qm))
260          h=new_jfm_glue(ps,pm,
261                             ltj.find_char_type('diffmet',pm),
262                             has_attr(p,attr_jchar_class))
263          g=ltj.calc_between_two_jchar_aux(g,h)
264       end
265    end
266    return g
267 end
268
269
270 -- In the beginning of a hbox created by line breaking, there are the followings:
271 --   o a hbox by \parindent
272 --   o a whatsit node which contains local paragraph materials.
273 -- When we insert jfm glues, we ignore these nodes.
274 function ltj.is_parindent_box(p)
275    if node_type(p.id)=='hlist' then 
276       return (p.subtype==3)
277       -- hlist (subtype=3) is a box by \parindent
278    elseif p.id==id_whatsit then 
279       return (p.subtype==node.subtype('local_par'))
280    end
281 end
282
283 local function add_kinsoku_penalty(head,p)
284    local c = p.char
285    local e = get_penalty_table('pre',c)
286    if e~=0 then
287       local q = node.prev(p)
288       if q and node_type(q.id)=='penalty' then
289          q.penalty=q.penalty+e
290       else 
291          q=node_new(node.id('penalty'))
292          q.penalty=e
293          node_insert_before(head,p,q)
294       end
295    end
296    e = get_penalty_table('post',c)
297    if e~=0 then
298       local q = next_node(p)
299       if q and node_type(q.id)=='penalty' then
300          q.penalty=q.penalty+e
301          return false
302       else 
303          q=node_new(node.id('penalty'))
304          q.penalty=e
305          node_insert_after(head,p,q)
306          return true
307       end
308    end
309 end
310
311 -- Insert jfm glue: main routine
312
313 local function insert_jfm_glue(head)
314    local p = head
315    local q = nil  -- the previous node of p
316    local g
317    local ihb_flag = false
318    local inserted_after_penalty = false
319    if not p then 
320       return head 
321    end
322    while p and  ltj.is_parindent_box(p) do p=next_node(p) end
323    while p do
324       if p.id==id_whatsit and p.subtype==node.subtype('user_defined')
325          and p.user_id==30111 then
326          g=p; p=next_node(p); 
327          ihb_flag=true; head,p=node.remove(head, g)
328       else
329          g=calc_between_two_jchar(q,p)
330          if g and (not ihb_flag) then
331             h = node_insert_before(head,p,g)
332             if not q then head=h end 
333             -- If p is the first node (=head), the skip is inserted
334             -- before head. So we must change head.
335          end
336          --if is_japanese_glyph_node(q) then
337          --   node.insert(q, inserted_after_penalty)
338          --end
339          q=p; ihb_flag=false; 
340          if is_japanese_glyph_node(p) 
341             and add_kinsoku_penalty(head,p) then
342             p=next_node(p); inserted_after_penalty = true
343          else 
344             inserted_after_penalty = false
345          end
346          p=next_node(p)
347       end
348    end
349    -- Insert skip after the last node
350    g=calc_between_two_jchar(q,nil)
351    if g then h = node_insert_after(head,q,g) end
352    return head
353 end
354
355
356
357 -- Insert \xkanjiskip at the boundaries between Japanese characters 
358 -- and non-Japanese characters. 
359 -- We also insert \kanjiskip between Kanji in this function.
360 local kanji_skip={}
361 local xkanji_skip={}
362 local cx = nil
363 local no_skip=0
364 local after_schar=1
365 local after_wchar=2
366 local insert_skip=no_skip
367
368
369 -- In the next two function, cx is the Kanji code.
370 local function insert_akxsp(head,q)
371    if get_inhibit_xsp_table(cx)<=1 then return end
372    local g = node_new(id_glue)
373    g.subtype=0; g.spec=node.copy(xkanji_skip)
374    node_insert_after(head,q,g)
375 end
376
377 local function insert_kaxsp(head,q,p)
378    local g=true
379    local c=p.char
380    while p.components and p.subtype 
381       and math.floor(p.subtype/2)%2==1 do
382       p=p.components; c = p.char
383    end
384    if get_inhibit_xsp_table(c)%2 == 1 then
385       if get_inhibit_xsp_table(cx)%2==0 then g=false end
386    else 
387       g=false
388    end
389    if g then
390       g = node_new(id_glue)
391       g.subtype=0; g.spec=node.copy(xkanji_skip)
392       node_insert_after(head,q,g)
393    end
394 end
395
396
397 local function set_insert_skip_after_achar(p)
398    local c=p.char
399    while p.components and p.subtype 
400       and math.floor(p.subtype/2)%2==1 do
401       p=node.tail(p.components); c = p.char
402    end
403   if get_inhibit_xsp_table(c)>=2 then
404      insert_skip=after_schar
405   else
406      insert_skip=no_skip
407   end
408 end
409
410 -- Insert \xkanjiskip before p, a glyph node
411 local function insks_around_char(head,q,p)
412    if is_japanese_glyph_node(p) then
413       cx=p.char
414       if is_japanese_glyph_node(q)  then
415          local g = node_new(id_glue)
416          g.subtype=0; g.spec=node.copy(kanji_skip)
417          node_insert_before(head,p,g)
418       elseif insert_skip==after_schar then
419          insert_akxsp(head,q)
420       end
421       insert_skip=after_wchar
422    else
423       if insert_skip==after_wchar then
424          insert_kaxsp(head,q,p)
425       end
426       set_insert_skip_after_achar(p)
427    end
428 end
429
430 -- Return first and last glyph nodes in a hbox
431 local first_char = nil
432 local last_char = nil
433 local find_first_char = nil
434 local function check_box(bp)
435    local p = bp; local  flag = false
436    while p do
437       local pt = node_type(p.id)
438       if pt=='glyph' then
439          repeat 
440             if find_first_char then
441                first_char=p; find_first_char=false
442             end
443             last_char=p; flag=true; p=next_node(p)
444             if not p then return flag end
445          until p.id~=id_glyph
446       end
447       if pt=='hlist' then
448          flag=true
449          if p.shift==0 then
450             if check_box(p.head) then flag=true end
451          else if find_first_char then 
452                find_first_char=false
453             else 
454                last_char=nil
455             end
456          end
457       elseif pt == 'ins' or pt == 'mark'
458          or pt == 'adjust' 
459          or pt == 'whatsit' or pt == 'penalty' then
460          p=p
461       else
462          flag=true
463          if find_first_char then 
464             find_first_char=false
465          else 
466             last_char=nil
467          end
468       end
469       p=next_node(p)
470    end
471    return flag
472 end 
473
474 -- Insert \xkanjiskip around p, an hbox
475 local function insks_around_hbox(head,q,p)
476    if p.shift==0 then
477       find_first_char=true; first_char=nil; last_char=nil
478       if check_box(p.head) then
479          -- first char
480          if is_japanese_glyph_node(first_char) then
481             cx=first_char.char
482             if insert_skip==after_schar then 
483                insert_akxsp(head,q)
484             elseif insert_skip==after_wchar then
485                local g = node_new(id_glue)
486                g.subtype=0; g.spec=node.copy(kanji_skip)
487                node_insert_before(head,p,g)
488             end
489             insert_skip=after_wchar
490          elseif first_char then
491             cx=first_char.char
492             if insert_skip==after_wchar then
493                insert_kaxsp(head,q,first_char)
494             end
495             set_insert_skip_after_achar(first_char)
496          end
497          -- last char
498          if is_japanese_glyph_node(last_char) then
499             if is_japanese_glyph_node(next_node(p)) then
500                local g = node_new(id_glue)
501                g.subtype=0; g.spec=node.copy(kanji_skip)
502                node_insert_after(head,p,g)
503             end
504             insert_skip=after_wchar
505          elseif last_char then
506             set_insert_skip_after_achar(last_char)
507          else insert_skip=no_skip
508          end
509       else insert_skip=no_skip
510       end
511    else insert_skip=no_skip
512    end
513 end
514
515 -- Insert \xkanjiskip around p, a penalty
516 local function insks_around_penalty(head,q,p)
517    local r=next_node(p)
518    if r  and r.id==id_glyph then
519       if is_japanese_glyph_node(r) then
520          cx=r.char
521          if is_japanese_glyph_node(p)  then
522             local g = node_new(id_glue)
523             g.subtype=0; g.spec=node.copy(kanji_skip)
524             node_insert_before(head,r,g)
525          elseif insert_skip==insert_schar then
526             insert_akxsp(head,p)
527          end
528          q=p; p=next_node(p)
529          insert_skip=after_wchar
530       else
531          if insert_skip==after_wchar then
532             insert_kaxsp(head,p,r)
533          end
534          set_insert_skip_after_achar(r)
535       end
536    end
537 end
538
539 -- Insert \xkanjiskip around p, a kern
540 local function insks_around_kern(head,q,p)
541    if p.subtype==1 then -- \kern or \/
542       if not has_attr(p,luatexbase.attributes['luatexja@icflag']) then
543          insert_skip=no_skip
544       end
545    elseif p.subtype==2 then -- \accent: We ignore the accent character.
546       local v = next_node(next_node(next_node(p)))
547       if v and v.id==id_glyph then
548          insks_around_char(head,q,v)
549       end
550    end
551 end
552
553 -- Insert \xkanjiskip around p, a math_node
554 local function insks_around_math(head,q,p)
555    local g = { char = -1 }
556    if (p.subtype==0) and (insert_skip==after_wchar) then
557       insert_kaxsp(head,q,g)
558       insert_skip=no_skip
559    else
560       set_insert_skip_after_achar(g)
561    end
562 end
563
564 local function insert_kanji_skip(head)
565    if ltj.auto_spacing then
566       kanji_skip=tex.skip['kanjiskip']
567    else
568       kanji_skip=node_new(id_glue_spec)
569       kanji_skip.width=0;  kanji_skip.stretch=0; kanji_skip.shrink=0
570    end
571    if ltj.auto_xspacing then
572       xkanji_skip=tex.skip['xkanjiskip']
573    else
574       xkanji_skip=node_new(id_glue_spec)
575       xkanji_skip.width=0;  xkanji_skip.stretch=0; xkanji_skip.shrink=0
576    end
577    local p=head -- 「現在のnode」
578    local q=nil  -- pの一つ前 
579    insert_skip=no_skip
580    while p do
581       local pt = node_type(p.id)
582       if pt=='glyph' then
583          repeat 
584             insks_around_char(head,q,p)
585             q=p; p=next_node(p)
586          until (not p) or p.id~=id_glyph
587       else
588          if pt == 'hlist' then
589             insks_around_hbox(head,q,p)
590          elseif pt == 'penalty' then
591             insks_around_penalty(head,q,p)
592          elseif pt == 'kern' then
593             insks_around_kern(head,q,p)
594          elseif pt == 'math' then
595             insks_around_math(head,q,p)
596          elseif pt == 'ins' or pt == 'mark'
597             or pt == 'adjust'
598             or pt == 'whatsit' then
599             -- do nothing
600             p=p
601          else
602             -- rule, disc, glue, margin_kern
603             insert_skip=no_skip
604          end
605          q=p; p=next_node(p)
606       end
607    end
608    return head
609 end
610
611 -- Shift baseline
612 local function baselineshift(head)
613    local p=head
614    local m=false -- is in math mode?
615    while p do 
616       local v=has_attr(p,attr_yablshift)
617       if v then
618          local pt = node_type(p.id)
619          if pt=='glyph' then
620             p.yoffset=p.yoffset-v
621          elseif pt=='math' then
622             m=(p.subtype==0)
623          end
624          if m then -- boxes and rules are shifted only in math mode
625             if pt=='hlist' or pt=='vlist' then
626                p.shift=p.shift+v
627             elseif pt=='rule' then
628                p.height=p.height-v; p.depth=p.depth+v 
629             end
630          end
631       end
632       p=next_node(p)
633    end
634    return head
635 end
636
637
638 --====== Adjust the width of Japanese glyphs
639
640 -- TeX's \hss
641 local function get_hss()
642    local hss = node_new(id_glue)
643    local hss_spec = node_new(id_glue_spec)
644    hss_spec.width = 0
645    hss_spec.stretch = 65536
646    hss_spec.stretch_order = 2
647    hss_spec.shrink = 65536
648    hss_spec.shrink_order = 2
649    hss.spec = hss_spec
650    return hss
651 end
652
653 local function set_ja_width(head)
654    local p = head
655    local t,s,th, g, q,a
656    while p do
657       if is_japanese_glyph_node(p) then
658          t=ltj.metrics[ltj.font_metric_table[p.font].jfm]
659          s=t.char_type[has_attr(p,attr_jchar_class)]
660          if not(s.left==0.0 and s.down==0.0 
661                 and round(s.width*ltj.font_metric_table[p.font].size)==p.width) then
662             -- must be encapsuled by a \hbox
663             head, q = node.remove(head,p)
664             p.next=nil
665             p.yoffset=round(p.yoffset-ltj.font_metric_table[p.font].size*s.down)
666             p.xoffset=round(p.xoffset-ltj.font_metric_table[p.font].size*s.left)
667             node_insert_after(p,p,get_hss())
668             g=node_hpack(p, round(ltj.font_metric_table[p.font].size*s.width)
669                          , 'exactly')
670             g.height=round(ltj.font_metric_table[p.font].size*s.height)
671             g.depth=round(ltj.font_metric_table[p.font].size*s.depth)
672             head,p = node_insert_before(head,q,g)
673             p=q
674          else p=next_node(p)
675          end
676       else p=next_node(p)
677       end
678    end
679    return head
680 end
681
682 -- main process
683 local function main_process(head)
684    local p = head
685    p = insert_jfm_glue(p)
686    p = insert_kanji_skip(p)
687    p = baselineshift(p)
688    p = set_ja_width(p)
689    return p
690 end
691
692 -- debug
693 local depth=""
694 function ltj.show_node_list(head)
695    local p =head; local k = depth
696    depth=depth .. '.'
697    while p do
698       local pt=node_type(p.id)
699       if pt == 'glyph' then
700          print(depth .. ' glyph', p.subtype, utf.char(p.char), p.font)
701       elseif pt=='hlist' then
702          print(depth .. ' hlist', p.subtype, '(' .. print_scaled(p.height)
703             .. '+' .. print_scaled(p.depth)
704          .. ')x' .. print_scaled(p.width) )
705          ltj.show_node_list(p.head)
706          depth=k
707       elseif pt == 'whatsit' then
708          print(depth .. ' whatsit', p.subtype)
709       elseif pt == 'glue' then
710          print(depth .. ' glue', p.subtype, print_spec(p.spec))
711       else
712          print(depth .. ' ' .. s, s.subtype)
713       end
714       p=next_node(p)
715    end
716 end
717
718
719
720 --- the following function is modified from jafontspec.lua (by K. Maeda).
721 --- Instead of "%", we use U+FFFFF for suppressing spaces.
722 local function process_input_buffer(buffer)
723    local c = utf.byte(buffer, utf.len(buffer))
724    local p = node.new(id_glyph)
725    p.char = c
726    if utf.len(buffer) > 0 
727    and ltj.is_ucs_in_japanese_char(p) then
728         buffer = buffer .. string.char(0xF3,0xBF,0xBF,0xBF) -- U+FFFFF
729    end
730    return buffer
731 end
732
733
734 ---------- Hyphenate
735 local function suppress_hyphenate_ja(head)
736    local p
737    for p in node.traverse(head) do
738       if p.id == id_glyph then
739          local pc=p.char
740          if ltj.is_ucs_in_japanese_char(p) then
741             local v = has_attr(p,attr_curjfnt)
742             if v then 
743                p.font=v 
744                local l=ltj.find_char_type(pc,ltj.font_metric_table[v].jfm) or 0
745                node.set_attribute(p,attr_jchar_class,l)
746             end
747             v=has_attr(p,luatexbase.attributes['luatexja@ykblshift'])
748             if v then 
749                node.set_attribute(p,attr_yablshift,v)
750             else
751                node.unset_attribute(p,attr_yablshift)
752             end
753             p.lang=ltj.ja_lang_number
754          end
755       end
756    end
757    lang.hyphenate(head)
758    return head -- 共通化のため値を返す
759 end
760
761 -- callbacks
762 luatexbase.add_to_callback('process_input_buffer', 
763    function (buffer)
764      return process_input_buffer(buffer)
765    end,'ltj.process_input_buffer')
766
767 luatexbase.add_to_callback('pre_linebreak_filter', 
768    function (head,groupcode)
769      return main_process(head)
770    end,'ltj.pre_linebreak_filter',2)
771 luatexbase.add_to_callback('hpack_filter', 
772   function (head,groupcode,size,packtype)
773      return main_process(head)
774   end,'ltj.hpack_filter',2)
775
776 --insert before callbacks from luaotfload
777 luatexbase.add_to_callback('hpack_filter', 
778   function (head,groupcode,size,packtype)
779      return suppress_hyphenate_ja(head)
780   end,'ltj.hpack_filter_pre',0)
781 luatexbase.add_to_callback('hyphenate', 
782  function (head,tail)
783     return suppress_hyphenate_ja(head)
784  end,'ltj.hyphenate')