Compare commits
3 Commits
main
..
docs-static
| Author | SHA1 | Date | |
|---|---|---|---|
| a186cfbbf2 | |||
| a3276d75a3 | |||
| 0482ded1f7 |
@@ -1,3 +0,0 @@
|
||||
site/
|
||||
venv
|
||||
.venv
|
||||
|
After Width: | Height: | Size: 1.8 KiB |
@@ -0,0 +1,18 @@
|
||||
/*!
|
||||
* Lunr languages, `Danish` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.da=function(){this.pipeline.reset(),this.pipeline.add(e.da.trimmer,e.da.stopWordFilter,e.da.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.da.stemmer))},e.da.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.da.trimmer=e.trimmerSupport.generateTrimmer(e.da.wordCharacters),e.Pipeline.registerFunction(e.da.trimmer,"trimmer-da"),e.da.stemmer=function(){var r=e.stemmerSupport.Among,i=e.stemmerSupport.SnowballProgram,n=new function(){function e(){var e,r=f.cursor+3;if(d=f.limit,0<=r&&r<=f.limit){for(a=r;;){if(e=f.cursor,f.in_grouping(w,97,248)){f.cursor=e;break}if(f.cursor=e,e>=f.limit)return;f.cursor++}for(;!f.out_grouping(w,97,248);){if(f.cursor>=f.limit)return;f.cursor++}d=f.cursor,d<a&&(d=a)}}function n(){var e,r;if(f.cursor>=d&&(r=f.limit_backward,f.limit_backward=d,f.ket=f.cursor,e=f.find_among_b(c,32),f.limit_backward=r,e))switch(f.bra=f.cursor,e){case 1:f.slice_del();break;case 2:f.in_grouping_b(p,97,229)&&f.slice_del()}}function t(){var e,r=f.limit-f.cursor;f.cursor>=d&&(e=f.limit_backward,f.limit_backward=d,f.ket=f.cursor,f.find_among_b(l,4)?(f.bra=f.cursor,f.limit_backward=e,f.cursor=f.limit-r,f.cursor>f.limit_backward&&(f.cursor--,f.bra=f.cursor,f.slice_del())):f.limit_backward=e)}function s(){var e,r,i,n=f.limit-f.cursor;if(f.ket=f.cursor,f.eq_s_b(2,"st")&&(f.bra=f.cursor,f.eq_s_b(2,"ig")&&f.slice_del()),f.cursor=f.limit-n,f.cursor>=d&&(r=f.limit_backward,f.limit_backward=d,f.ket=f.cursor,e=f.find_among_b(m,5),f.limit_backward=r,e))switch(f.bra=f.cursor,e){case 1:f.slice_del(),i=f.limit-f.cursor,t(),f.cursor=f.limit-i;break;case 2:f.slice_from("løs")}}function o(){var e;f.cursor>=d&&(e=f.limit_backward,f.limit_backward=d,f.ket=f.cursor,f.out_grouping_b(w,97,248)?(f.bra=f.cursor,u=f.slice_to(u),f.limit_backward=e,f.eq_v_b(u)&&f.slice_del()):f.limit_backward=e)}var a,d,u,c=[new r("hed",-1,1),new r("ethed",0,1),new r("ered",-1,1),new r("e",-1,1),new r("erede",3,1),new r("ende",3,1),new r("erende",5,1),new r("ene",3,1),new r("erne",3,1),new r("ere",3,1),new r("en",-1,1),new r("heden",10,1),new r("eren",10,1),new r("er",-1,1),new r("heder",13,1),new r("erer",13,1),new r("s",-1,2),new r("heds",16,1),new r("es",16,1),new r("endes",18,1),new r("erendes",19,1),new r("enes",18,1),new r("ernes",18,1),new r("eres",18,1),new r("ens",16,1),new r("hedens",24,1),new r("erens",24,1),new r("ers",16,1),new r("ets",16,1),new r("erets",28,1),new r("et",-1,1),new r("eret",30,1)],l=[new r("gd",-1,-1),new r("dt",-1,-1),new r("gt",-1,-1),new r("kt",-1,-1)],m=[new r("ig",-1,1),new r("lig",0,1),new r("elig",1,1),new r("els",-1,1),new r("løst",-1,2)],w=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,48,0,128],p=[239,254,42,3,0,0,0,0,0,0,0,0,0,0,0,0,16],f=new i;this.setCurrent=function(e){f.setCurrent(e)},this.getCurrent=function(){return f.getCurrent()},this.stem=function(){var r=f.cursor;return e(),f.limit_backward=r,f.cursor=f.limit,n(),f.cursor=f.limit,t(),f.cursor=f.limit,s(),f.cursor=f.limit,o(),!0}};return function(e){return"function"==typeof e.update?e.update(function(e){return n.setCurrent(e),n.stem(),n.getCurrent()}):(n.setCurrent(e),n.stem(),n.getCurrent())}}(),e.Pipeline.registerFunction(e.da.stemmer,"stemmer-da"),e.da.stopWordFilter=e.generateStopWordFilter("ad af alle alt anden at blev blive bliver da de dem den denne der deres det dette dig din disse dog du efter eller en end er et for fra ham han hans har havde have hende hendes her hos hun hvad hvis hvor i ikke ind jeg jer jo kunne man mange med meget men mig min mine mit mod ned noget nogle nu når og også om op os over på selv sig sin sine sit skal skulle som sådan thi til ud under var vi vil ville vor være været".split(" ")),e.Pipeline.registerFunction(e.da.stopWordFilter,"stopWordFilter-da")}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.hi=function(){this.pipeline.reset(),this.pipeline.add(e.hi.trimmer,e.hi.stopWordFilter,e.hi.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.hi.stemmer))},e.hi.wordCharacters="ऀ-ःऄ-एऐ-टठ-यर-िी-ॏॐ-य़ॠ-९॰-ॿa-zA-Za-zA-Z0-90-9",e.hi.trimmer=e.trimmerSupport.generateTrimmer(e.hi.wordCharacters),e.Pipeline.registerFunction(e.hi.trimmer,"trimmer-hi"),e.hi.stopWordFilter=e.generateStopWordFilter("अत अपना अपनी अपने अभी अंदर आदि आप इत्यादि इन इनका इन्हीं इन्हें इन्हों इस इसका इसकी इसके इसमें इसी इसे उन उनका उनकी उनके उनको उन्हीं उन्हें उन्हों उस उसके उसी उसे एक एवं एस ऐसे और कई कर करता करते करना करने करें कहते कहा का काफ़ी कि कितना किन्हें किन्हों किया किर किस किसी किसे की कुछ कुल के को कोई कौन कौनसा गया घर जब जहाँ जा जितना जिन जिन्हें जिन्हों जिस जिसे जीधर जैसा जैसे जो तक तब तरह तिन तिन्हें तिन्हों तिस तिसे तो था थी थे दबारा दिया दुसरा दूसरे दो द्वारा न नके नहीं ना निहायत नीचे ने पर पहले पूरा पे फिर बनी बही बहुत बाद बाला बिलकुल भी भीतर मगर मानो मे में यदि यह यहाँ यही या यिह ये रखें रहा रहे ऱ्वासा लिए लिये लेकिन व वग़ैरह वर्ग वह वहाँ वहीं वाले वुह वे वो सकता सकते सबसे सभी साथ साबुत साभ सारा से सो संग ही हुआ हुई हुए है हैं हो होता होती होते होना होने".split(" ")),e.hi.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var r=e.wordcut;r.init(),e.hi.tokenizer=function(i){if(!arguments.length||null==i||void 0==i)return[];if(Array.isArray(i))return i.map(function(r){return isLunr2?new e.Token(r.toLowerCase()):r.toLowerCase()});var t=i.toString().toLowerCase().replace(/^\s+/,"");return r.cut(t).split("|")},e.Pipeline.registerFunction(e.hi.stemmer,"stemmer-hi"),e.Pipeline.registerFunction(e.hi.stopWordFilter,"stopWordFilter-hi")}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.hy=function(){this.pipeline.reset(),this.pipeline.add(e.hy.trimmer,e.hy.stopWordFilter)},e.hy.wordCharacters="[A-Za-z-֏ff-ﭏ]",e.hy.trimmer=e.trimmerSupport.generateTrimmer(e.hy.wordCharacters),e.Pipeline.registerFunction(e.hy.trimmer,"trimmer-hy"),e.hy.stopWordFilter=e.generateStopWordFilter("դու և եք էիր էիք հետո նաև նրանք որը վրա է որ պիտի են այս մեջ ն իր ու ի այդ որոնք այն կամ էր մի ես համար այլ իսկ էին ենք հետ ին թ էինք մենք նրա նա դուք եմ էի ըստ որպես ում".split(" ")),e.Pipeline.registerFunction(e.hy.stopWordFilter,"stopWordFilter-hy"),e.hy.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}(),e.Pipeline.registerFunction(e.hy.stemmer,"stemmer-hy")}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var r="2"==e.version[0];e.ja=function(){this.pipeline.reset(),this.pipeline.add(e.ja.trimmer,e.ja.stopWordFilter,e.ja.stemmer),r?this.tokenizer=e.ja.tokenizer:(e.tokenizer&&(e.tokenizer=e.ja.tokenizer),this.tokenizerFn&&(this.tokenizerFn=e.ja.tokenizer))};var t=new e.TinySegmenter;e.ja.tokenizer=function(i){var n,o,s,p,a,u,m,l,c,f;if(!arguments.length||null==i||void 0==i)return[];if(Array.isArray(i))return i.map(function(t){return r?new e.Token(t.toLowerCase()):t.toLowerCase()});for(o=i.toString().toLowerCase().replace(/^\s+/,""),n=o.length-1;n>=0;n--)if(/\S/.test(o.charAt(n))){o=o.substring(0,n+1);break}for(a=[],s=o.length,c=0,l=0;c<=s;c++)if(u=o.charAt(c),m=c-l,u.match(/\s/)||c==s){if(m>0)for(p=t.segment(o.slice(l,c)).filter(function(e){return!!e}),f=l,n=0;n<p.length;n++)r?a.push(new e.Token(p[n],{position:[f,p[n].length],index:a.length})):a.push(p[n]),f+=p[n].length;l=c+1}return a},e.ja.stemmer=function(){return function(e){return e}}(),e.Pipeline.registerFunction(e.ja.stemmer,"stemmer-ja"),e.ja.wordCharacters="一二三四五六七八九十百千万億兆一-龠々〆ヵヶぁ-んァ-ヴーア-ン゙a-zA-Za-zA-Z0-90-9",e.ja.trimmer=e.trimmerSupport.generateTrimmer(e.ja.wordCharacters),e.Pipeline.registerFunction(e.ja.trimmer,"trimmer-ja"),e.ja.stopWordFilter=e.generateStopWordFilter("これ それ あれ この その あの ここ そこ あそこ こちら どこ だれ なに なん 何 私 貴方 貴方方 我々 私達 あの人 あのかた 彼女 彼 です あります おります います は が の に を で え から まで より も どの と し それで しかし".split(" ")),e.Pipeline.registerFunction(e.ja.stopWordFilter,"stopWordFilter-ja"),e.jp=e.ja,e.Pipeline.registerFunction(e.jp.stemmer,"stemmer-jp"),e.Pipeline.registerFunction(e.jp.trimmer,"trimmer-jp"),e.Pipeline.registerFunction(e.jp.stopWordFilter,"stopWordFilter-jp")}});
|
||||
@@ -0,0 +1 @@
|
||||
module.exports=require("./lunr.ja");
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.kn=function(){this.pipeline.reset(),this.pipeline.add(e.kn.trimmer,e.kn.stopWordFilter,e.kn.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.kn.stemmer))},e.kn.wordCharacters="ಀ-಄ಅ-ಔಕ-ಹಾ-ೌ಼-ಽೕ-ೖೝ-ೞೠ-ೡೢ-ೣ೦-೯ೱ-ೳ",e.kn.trimmer=e.trimmerSupport.generateTrimmer(e.kn.wordCharacters),e.Pipeline.registerFunction(e.kn.trimmer,"trimmer-kn"),e.kn.stopWordFilter=e.generateStopWordFilter("ಮತ್ತು ಈ ಒಂದು ರಲ್ಲಿ ಹಾಗೂ ಎಂದು ಅಥವಾ ಇದು ರ ಅವರು ಎಂಬ ಮೇಲೆ ಅವರ ತನ್ನ ಆದರೆ ತಮ್ಮ ನಂತರ ಮೂಲಕ ಹೆಚ್ಚು ನ ಆ ಕೆಲವು ಅನೇಕ ಎರಡು ಹಾಗು ಪ್ರಮುಖ ಇದನ್ನು ಇದರ ಸುಮಾರು ಅದರ ಅದು ಮೊದಲ ಬಗ್ಗೆ ನಲ್ಲಿ ರಂದು ಇತರ ಅತ್ಯಂತ ಹೆಚ್ಚಿನ ಸಹ ಸಾಮಾನ್ಯವಾಗಿ ನೇ ಹಲವಾರು ಹೊಸ ದಿ ಕಡಿಮೆ ಯಾವುದೇ ಹೊಂದಿದೆ ದೊಡ್ಡ ಅನ್ನು ಇವರು ಪ್ರಕಾರ ಇದೆ ಮಾತ್ರ ಕೂಡ ಇಲ್ಲಿ ಎಲ್ಲಾ ವಿವಿಧ ಅದನ್ನು ಹಲವು ರಿಂದ ಕೇವಲ ದ ದಕ್ಷಿಣ ಗೆ ಅವನ ಅತಿ ನೆಯ ಬಹಳ ಕೆಲಸ ಎಲ್ಲ ಪ್ರತಿ ಇತ್ಯಾದಿ ಇವು ಬೇರೆ ಹೀಗೆ ನಡುವೆ ಇದಕ್ಕೆ ಎಸ್ ಇವರ ಮೊದಲು ಶ್ರೀ ಮಾಡುವ ಇದರಲ್ಲಿ ರೀತಿಯ ಮಾಡಿದ ಕಾಲ ಅಲ್ಲಿ ಮಾಡಲು ಅದೇ ಈಗ ಅವು ಗಳು ಎ ಎಂಬುದು ಅವನು ಅಂದರೆ ಅವರಿಗೆ ಇರುವ ವಿಶೇಷ ಮುಂದೆ ಅವುಗಳ ಮುಂತಾದ ಮೂಲ ಬಿ ಮೀ ಒಂದೇ ಇನ್ನೂ ಹೆಚ್ಚಾಗಿ ಮಾಡಿ ಅವರನ್ನು ಇದೇ ಯ ರೀತಿಯಲ್ಲಿ ಜೊತೆ ಅದರಲ್ಲಿ ಮಾಡಿದರು ನಡೆದ ಆಗ ಮತ್ತೆ ಪೂರ್ವ ಆತ ಬಂದ ಯಾವ ಒಟ್ಟು ಇತರೆ ಹಿಂದೆ ಪ್ರಮಾಣದ ಗಳನ್ನು ಕುರಿತು ಯು ಆದ್ದರಿಂದ ಅಲ್ಲದೆ ನಗರದ ಮೇಲಿನ ಏಕೆಂದರೆ ರಷ್ಟು ಎಂಬುದನ್ನು ಬಾರಿ ಎಂದರೆ ಹಿಂದಿನ ಆದರೂ ಆದ ಸಂಬಂಧಿಸಿದ ಮತ್ತೊಂದು ಸಿ ಆತನ ".split(" ")),e.kn.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var r=e.wordcut;r.init(),e.kn.tokenizer=function(t){if(!arguments.length||null==t||void 0==t)return[];if(Array.isArray(t))return t.map(function(r){return isLunr2?new e.Token(r.toLowerCase()):r.toLowerCase()});var n=t.toString().toLowerCase().replace(/^\s+/,"");return r.cut(n).split("|")},e.Pipeline.registerFunction(e.kn.stemmer,"stemmer-kn"),e.Pipeline.registerFunction(e.kn.stopWordFilter,"stopWordFilter-kn")}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(e){e.multiLanguage=function(){for(var t=Array.prototype.slice.call(arguments),i=t.join("-"),r="",n=[],s=[],p=0;p<t.length;++p)"en"==t[p]?(r+="\\w",n.unshift(e.stopWordFilter),n.push(e.stemmer),s.push(e.stemmer)):(r+=e[t[p]].wordCharacters,e[t[p]].stopWordFilter&&n.unshift(e[t[p]].stopWordFilter),e[t[p]].stemmer&&(n.push(e[t[p]].stemmer),s.push(e[t[p]].stemmer)));var o=e.trimmerSupport.generateTrimmer(r);return e.Pipeline.registerFunction(o,"lunr-multi-trimmer-"+i),n.unshift(o),function(){this.pipeline.reset(),this.pipeline.add.apply(this.pipeline,n),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add.apply(this.searchPipeline,s))}}}});
|
||||
@@ -0,0 +1,18 @@
|
||||
/*!
|
||||
* Lunr languages, `Norwegian` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.no=function(){this.pipeline.reset(),this.pipeline.add(e.no.trimmer,e.no.stopWordFilter,e.no.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.no.stemmer))},e.no.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.no.trimmer=e.trimmerSupport.generateTrimmer(e.no.wordCharacters),e.Pipeline.registerFunction(e.no.trimmer,"trimmer-no"),e.no.stemmer=function(){var r=e.stemmerSupport.Among,n=e.stemmerSupport.SnowballProgram,i=new function(){function e(){var e,r=w.cursor+3;if(a=w.limit,0<=r||r<=w.limit){for(s=r;;){if(e=w.cursor,w.in_grouping(d,97,248)){w.cursor=e;break}if(e>=w.limit)return;w.cursor=e+1}for(;!w.out_grouping(d,97,248);){if(w.cursor>=w.limit)return;w.cursor++}a=w.cursor,a<s&&(a=s)}}function i(){var e,r,n;if(w.cursor>=a&&(r=w.limit_backward,w.limit_backward=a,w.ket=w.cursor,e=w.find_among_b(m,29),w.limit_backward=r,e))switch(w.bra=w.cursor,e){case 1:w.slice_del();break;case 2:n=w.limit-w.cursor,w.in_grouping_b(c,98,122)?w.slice_del():(w.cursor=w.limit-n,w.eq_s_b(1,"k")&&w.out_grouping_b(d,97,248)&&w.slice_del());break;case 3:w.slice_from("er")}}function t(){var e,r=w.limit-w.cursor;w.cursor>=a&&(e=w.limit_backward,w.limit_backward=a,w.ket=w.cursor,w.find_among_b(u,2)?(w.bra=w.cursor,w.limit_backward=e,w.cursor=w.limit-r,w.cursor>w.limit_backward&&(w.cursor--,w.bra=w.cursor,w.slice_del())):w.limit_backward=e)}function o(){var e,r;w.cursor>=a&&(r=w.limit_backward,w.limit_backward=a,w.ket=w.cursor,e=w.find_among_b(l,11),e?(w.bra=w.cursor,w.limit_backward=r,1==e&&w.slice_del()):w.limit_backward=r)}var s,a,m=[new r("a",-1,1),new r("e",-1,1),new r("ede",1,1),new r("ande",1,1),new r("ende",1,1),new r("ane",1,1),new r("ene",1,1),new r("hetene",6,1),new r("erte",1,3),new r("en",-1,1),new r("heten",9,1),new r("ar",-1,1),new r("er",-1,1),new r("heter",12,1),new r("s",-1,2),new r("as",14,1),new r("es",14,1),new r("edes",16,1),new r("endes",16,1),new r("enes",16,1),new r("hetenes",19,1),new r("ens",14,1),new r("hetens",21,1),new r("ers",14,1),new r("ets",14,1),new r("et",-1,1),new r("het",25,1),new r("ert",-1,3),new r("ast",-1,1)],u=[new r("dt",-1,-1),new r("vt",-1,-1)],l=[new r("leg",-1,1),new r("eleg",0,1),new r("ig",-1,1),new r("eig",2,1),new r("lig",2,1),new r("elig",4,1),new r("els",-1,1),new r("lov",-1,1),new r("elov",7,1),new r("slov",7,1),new r("hetslov",9,1)],d=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,48,0,128],c=[119,125,149,1],w=new n;this.setCurrent=function(e){w.setCurrent(e)},this.getCurrent=function(){return w.getCurrent()},this.stem=function(){var r=w.cursor;return e(),w.limit_backward=r,w.cursor=w.limit,i(),w.cursor=w.limit,t(),w.cursor=w.limit,o(),!0}};return function(e){return"function"==typeof e.update?e.update(function(e){return i.setCurrent(e),i.stem(),i.getCurrent()}):(i.setCurrent(e),i.stem(),i.getCurrent())}}(),e.Pipeline.registerFunction(e.no.stemmer,"stemmer-no"),e.no.stopWordFilter=e.generateStopWordFilter("alle at av bare begge ble blei bli blir blitt både båe da de deg dei deim deira deires dem den denne der dere deres det dette di din disse ditt du dykk dykkar då eg ein eit eitt eller elles en enn er et ett etter for fordi fra før ha hadde han hans har hennar henne hennes her hjå ho hoe honom hoss hossen hun hva hvem hver hvilke hvilken hvis hvor hvordan hvorfor i ikke ikkje ikkje ingen ingi inkje inn inni ja jeg kan kom korleis korso kun kunne kva kvar kvarhelst kven kvi kvifor man mange me med medan meg meget mellom men mi min mine mitt mot mykje ned no noe noen noka noko nokon nokor nokre nå når og også om opp oss over på samme seg selv si si sia sidan siden sin sine sitt sjøl skal skulle slik so som som somme somt så sånn til um upp ut uten var vart varte ved vere verte vi vil ville vore vors vort vår være være vært å".split(" ")),e.Pipeline.registerFunction(e.no.stopWordFilter,"stopWordFilter-no")}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.sa=function(){this.pipeline.reset(),this.pipeline.add(e.sa.trimmer,e.sa.stopWordFilter,e.sa.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.sa.stemmer))},e.sa.wordCharacters="ऀ-ःऄ-एऐ-टठ-यर-िी-ॏॐ-य़ॠ-९॰-ॿ꣠-꣱ꣲ-ꣷ꣸-ꣻ꣼-ꣽꣾ-ꣿᆰ0-ᆰ9",e.sa.trimmer=e.trimmerSupport.generateTrimmer(e.sa.wordCharacters),e.Pipeline.registerFunction(e.sa.trimmer,"trimmer-sa"),e.sa.stopWordFilter=e.generateStopWordFilter('तथा अयम् एकम् इत्यस्मिन् तथा तत् वा अयम् इत्यस्य ते आहूत उपरि तेषाम् किन्तु तेषाम् तदा इत्यनेन अधिकः इत्यस्य तत् केचन बहवः द्वि तथा महत्वपूर्णः अयम् अस्य विषये अयं अस्ति तत् प्रथमः विषये इत्युपरि इत्युपरि इतर अधिकतमः अधिकः अपि सामान्यतया ठ इतरेतर नूतनम् द न्यूनम् कश्चित् वा विशालः द सः अस्ति तदनुसारम् तत्र अस्ति केवलम् अपि अत्र सर्वे विविधाः तत् बहवः यतः इदानीम् द दक्षिण इत्यस्मै तस्य उपरि नथ अतीव कार्यम् सर्वे एकैकम् इत्यादि। एते सन्ति उत इत्थम् मध्ये एतदर्थं . स कस्य प्रथमः श्री. करोति अस्मिन् प्रकारः निर्मिता कालः तत्र कर्तुं समान अधुना ते सन्ति स एकः अस्ति सः अर्थात् तेषां कृते . स्थितम् विशेषः अग्रिम तेषाम् समान स्रोतः ख म समान इदानीमपि अधिकतया करोतु ते समान इत्यस्य वीथी सह यस्मिन् कृतवान् धृतः तदा पुनः पूर्वं सः आगतः किम् कुल इतर पुरा मात्रा स विषये उ अतएव अपि नगरस्य उपरि यतः प्रतिशतं कतरः कालः साधनानि भूत तथापि जात सम्बन्धि अन्यत् ग अतः अस्माकं स्वकीयाः अस्माकं इदानीं अन्तः इत्यादयः भवन्तः इत्यादयः एते एताः तस्य अस्य इदम् एते तेषां तेषां तेषां तान् तेषां तेषां तेषां समानः सः एकः च तादृशाः बहवः अन्ये च वदन्ति यत् कियत् कस्मै कस्मै यस्मै यस्मै यस्मै यस्मै न अतिनीचः किन्तु प्रथमं सम्पूर्णतया ततः चिरकालानन्तरं पुस्तकं सम्पूर्णतया अन्तः किन्तु अत्र वा इह इव श्रद्धाय अवशिष्यते परन्तु अन्ये वर्गाः सन्ति ते सन्ति शक्नुवन्ति सर्वे मिलित्वा सर्वे एकत्र"'.split(" ")),e.sa.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var r=e.wordcut;r.init(),e.sa.tokenizer=function(t){if(!arguments.length||null==t||void 0==t)return[];if(Array.isArray(t))return t.map(function(r){return isLunr2?new e.Token(r.toLowerCase()):r.toLowerCase()});var i=t.toString().toLowerCase().replace(/^\s+/,"");return r.cut(i).split("|")},e.Pipeline.registerFunction(e.sa.stemmer,"stemmer-sa"),e.Pipeline.registerFunction(e.sa.stopWordFilter,"stopWordFilter-sa")}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(r,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(r.lunr)}(this,function(){return function(r){r.stemmerSupport={Among:function(r,t,i,s){if(this.toCharArray=function(r){for(var t=r.length,i=new Array(t),s=0;s<t;s++)i[s]=r.charCodeAt(s);return i},!r&&""!=r||!t&&0!=t||!i)throw"Bad Among initialisation: s:"+r+", substring_i: "+t+", result: "+i;this.s_size=r.length,this.s=this.toCharArray(r),this.substring_i=t,this.result=i,this.method=s},SnowballProgram:function(){var r;return{bra:0,ket:0,limit:0,cursor:0,limit_backward:0,setCurrent:function(t){r=t,this.cursor=0,this.limit=t.length,this.limit_backward=0,this.bra=this.cursor,this.ket=this.limit},getCurrent:function(){var t=r;return r=null,t},in_grouping:function(t,i,s){if(this.cursor<this.limit){var e=r.charCodeAt(this.cursor);if(e<=s&&e>=i&&(e-=i,t[e>>3]&1<<(7&e)))return this.cursor++,!0}return!1},in_grouping_b:function(t,i,s){if(this.cursor>this.limit_backward){var e=r.charCodeAt(this.cursor-1);if(e<=s&&e>=i&&(e-=i,t[e>>3]&1<<(7&e)))return this.cursor--,!0}return!1},out_grouping:function(t,i,s){if(this.cursor<this.limit){var e=r.charCodeAt(this.cursor);if(e>s||e<i)return this.cursor++,!0;if(e-=i,!(t[e>>3]&1<<(7&e)))return this.cursor++,!0}return!1},out_grouping_b:function(t,i,s){if(this.cursor>this.limit_backward){var e=r.charCodeAt(this.cursor-1);if(e>s||e<i)return this.cursor--,!0;if(e-=i,!(t[e>>3]&1<<(7&e)))return this.cursor--,!0}return!1},eq_s:function(t,i){if(this.limit-this.cursor<t)return!1;for(var s=0;s<t;s++)if(r.charCodeAt(this.cursor+s)!=i.charCodeAt(s))return!1;return this.cursor+=t,!0},eq_s_b:function(t,i){if(this.cursor-this.limit_backward<t)return!1;for(var s=0;s<t;s++)if(r.charCodeAt(this.cursor-t+s)!=i.charCodeAt(s))return!1;return this.cursor-=t,!0},find_among:function(t,i){for(var s=0,e=i,n=this.cursor,u=this.limit,o=0,h=0,c=!1;;){for(var a=s+(e-s>>1),f=0,l=o<h?o:h,_=t[a],m=l;m<_.s_size;m++){if(n+l==u){f=-1;break}if(f=r.charCodeAt(n+l)-_.s[m])break;l++}if(f<0?(e=a,h=l):(s=a,o=l),e-s<=1){if(s>0||e==s||c)break;c=!0}}for(;;){var _=t[s];if(o>=_.s_size){if(this.cursor=n+_.s_size,!_.method)return _.result;var b=_.method();if(this.cursor=n+_.s_size,b)return _.result}if((s=_.substring_i)<0)return 0}},find_among_b:function(t,i){for(var s=0,e=i,n=this.cursor,u=this.limit_backward,o=0,h=0,c=!1;;){for(var a=s+(e-s>>1),f=0,l=o<h?o:h,_=t[a],m=_.s_size-1-l;m>=0;m--){if(n-l==u){f=-1;break}if(f=r.charCodeAt(n-1-l)-_.s[m])break;l++}if(f<0?(e=a,h=l):(s=a,o=l),e-s<=1){if(s>0||e==s||c)break;c=!0}}for(;;){var _=t[s];if(o>=_.s_size){if(this.cursor=n-_.s_size,!_.method)return _.result;var b=_.method();if(this.cursor=n-_.s_size,b)return _.result}if((s=_.substring_i)<0)return 0}},replace_s:function(t,i,s){var e=s.length-(i-t),n=r.substring(0,t),u=r.substring(i);return r=n+s+u,this.limit+=e,this.cursor>=i?this.cursor+=e:this.cursor>t&&(this.cursor=t),e},slice_check:function(){if(this.bra<0||this.bra>this.ket||this.ket>this.limit||this.limit>r.length)throw"faulty slice operation"},slice_from:function(r){this.slice_check(),this.replace_s(this.bra,this.ket,r)},slice_del:function(){this.slice_from("")},insert:function(r,t,i){var s=this.replace_s(r,t,i);r<=this.bra&&(this.bra+=s),r<=this.ket&&(this.ket+=s)},slice_to:function(){return this.slice_check(),r.substring(this.bra,this.ket)},eq_v_b:function(r){return this.eq_s_b(r.length,r)}}}},r.trimmerSupport={generateTrimmer:function(r){var t=new RegExp("^[^"+r+"]+"),i=new RegExp("[^"+r+"]+$");return function(r){return"function"==typeof r.update?r.update(function(r){return r.replace(t,"").replace(i,"")}):r.replace(t,"").replace(i,"")}}}}});
|
||||
@@ -0,0 +1,18 @@
|
||||
/*!
|
||||
* Lunr languages, `Swedish` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.sv=function(){this.pipeline.reset(),this.pipeline.add(e.sv.trimmer,e.sv.stopWordFilter,e.sv.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.sv.stemmer))},e.sv.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.sv.trimmer=e.trimmerSupport.generateTrimmer(e.sv.wordCharacters),e.Pipeline.registerFunction(e.sv.trimmer,"trimmer-sv"),e.sv.stemmer=function(){var r=e.stemmerSupport.Among,n=e.stemmerSupport.SnowballProgram,t=new function(){function e(){var e,r=w.cursor+3;if(o=w.limit,0<=r||r<=w.limit){for(a=r;;){if(e=w.cursor,w.in_grouping(l,97,246)){w.cursor=e;break}if(w.cursor=e,w.cursor>=w.limit)return;w.cursor++}for(;!w.out_grouping(l,97,246);){if(w.cursor>=w.limit)return;w.cursor++}o=w.cursor,o<a&&(o=a)}}function t(){var e,r=w.limit_backward;if(w.cursor>=o&&(w.limit_backward=o,w.cursor=w.limit,w.ket=w.cursor,e=w.find_among_b(u,37),w.limit_backward=r,e))switch(w.bra=w.cursor,e){case 1:w.slice_del();break;case 2:w.in_grouping_b(d,98,121)&&w.slice_del()}}function i(){var e=w.limit_backward;w.cursor>=o&&(w.limit_backward=o,w.cursor=w.limit,w.find_among_b(c,7)&&(w.cursor=w.limit,w.ket=w.cursor,w.cursor>w.limit_backward&&(w.bra=--w.cursor,w.slice_del())),w.limit_backward=e)}function s(){var e,r;if(w.cursor>=o){if(r=w.limit_backward,w.limit_backward=o,w.cursor=w.limit,w.ket=w.cursor,e=w.find_among_b(m,5))switch(w.bra=w.cursor,e){case 1:w.slice_del();break;case 2:w.slice_from("lös");break;case 3:w.slice_from("full")}w.limit_backward=r}}var a,o,u=[new r("a",-1,1),new r("arna",0,1),new r("erna",0,1),new r("heterna",2,1),new r("orna",0,1),new r("ad",-1,1),new r("e",-1,1),new r("ade",6,1),new r("ande",6,1),new r("arne",6,1),new r("are",6,1),new r("aste",6,1),new r("en",-1,1),new r("anden",12,1),new r("aren",12,1),new r("heten",12,1),new r("ern",-1,1),new r("ar",-1,1),new r("er",-1,1),new r("heter",18,1),new r("or",-1,1),new r("s",-1,2),new r("as",21,1),new r("arnas",22,1),new r("ernas",22,1),new r("ornas",22,1),new r("es",21,1),new r("ades",26,1),new r("andes",26,1),new r("ens",21,1),new r("arens",29,1),new r("hetens",29,1),new r("erns",21,1),new r("at",-1,1),new r("andet",-1,1),new r("het",-1,1),new r("ast",-1,1)],c=[new r("dd",-1,-1),new r("gd",-1,-1),new r("nn",-1,-1),new r("dt",-1,-1),new r("gt",-1,-1),new r("kt",-1,-1),new r("tt",-1,-1)],m=[new r("ig",-1,1),new r("lig",0,1),new r("els",-1,1),new r("fullt",-1,3),new r("löst",-1,2)],l=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,24,0,32],d=[119,127,149],w=new n;this.setCurrent=function(e){w.setCurrent(e)},this.getCurrent=function(){return w.getCurrent()},this.stem=function(){var r=w.cursor;return e(),w.limit_backward=r,w.cursor=w.limit,t(),w.cursor=w.limit,i(),w.cursor=w.limit,s(),!0}};return function(e){return"function"==typeof e.update?e.update(function(e){return t.setCurrent(e),t.stem(),t.getCurrent()}):(t.setCurrent(e),t.stem(),t.getCurrent())}}(),e.Pipeline.registerFunction(e.sv.stemmer,"stemmer-sv"),e.sv.stopWordFilter=e.generateStopWordFilter("alla allt att av blev bli blir blivit de dem den denna deras dess dessa det detta dig din dina ditt du där då efter ej eller en er era ert ett från för ha hade han hans har henne hennes hon honom hur här i icke ingen inom inte jag ju kan kunde man med mellan men mig min mina mitt mot mycket ni nu när någon något några och om oss på samma sedan sig sin sina sitta själv skulle som så sådan sådana sådant till under upp ut utan vad var vara varför varit varje vars vart vem vi vid vilka vilkas vilken vilket vår våra vårt än är åt över".split(" ")),e.Pipeline.registerFunction(e.sv.stopWordFilter,"stopWordFilter-sv")}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.ta=function(){this.pipeline.reset(),this.pipeline.add(e.ta.trimmer,e.ta.stopWordFilter,e.ta.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.ta.stemmer))},e.ta.wordCharacters="-உஊ-ஏஐ-ஙச-ட-னப-யர-ஹ-ிீ-ொ-ௐ---௩௪-௯௰-௹௺-a-zA-Za-zA-Z0-90-9",e.ta.trimmer=e.trimmerSupport.generateTrimmer(e.ta.wordCharacters),e.Pipeline.registerFunction(e.ta.trimmer,"trimmer-ta"),e.ta.stopWordFilter=e.generateStopWordFilter("அங்கு அங்கே அது அதை அந்த அவர் அவர்கள் அவள் அவன் அவை ஆக ஆகவே ஆகையால் ஆதலால் ஆதலினால் ஆனாலும் ஆனால் இங்கு இங்கே இது இதை இந்த இப்படி இவர் இவர்கள் இவள் இவன் இவை இவ்வளவு உனக்கு உனது உன் உன்னால் எங்கு எங்கே எது எதை எந்த எப்படி எவர் எவர்கள் எவள் எவன் எவை எவ்வளவு எனக்கு எனது எனவே என் என்ன என்னால் ஏது ஏன் தனது தன்னால் தானே தான் நாங்கள் நாம் நான் நீ நீங்கள்".split(" ")),e.ta.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var t=e.wordcut;t.init(),e.ta.tokenizer=function(r){if(!arguments.length||null==r||void 0==r)return[];if(Array.isArray(r))return r.map(function(t){return isLunr2?new e.Token(t.toLowerCase()):t.toLowerCase()});var i=r.toString().toLowerCase().replace(/^\s+/,"");return t.cut(i).split("|")},e.Pipeline.registerFunction(e.ta.stemmer,"stemmer-ta"),e.Pipeline.registerFunction(e.ta.stopWordFilter,"stopWordFilter-ta")}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.te=function(){this.pipeline.reset(),this.pipeline.add(e.te.trimmer,e.te.stopWordFilter,e.te.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.te.stemmer))},e.te.wordCharacters="ఀ-ఄఅ-ఔక-హా-ౌౕ-ౖౘ-ౚౠ-ౡౢ-ౣ౦-౯౸-౿఼ఽ్ౝ౷",e.te.trimmer=e.trimmerSupport.generateTrimmer(e.te.wordCharacters),e.Pipeline.registerFunction(e.te.trimmer,"trimmer-te"),e.te.stopWordFilter=e.generateStopWordFilter("అందరూ అందుబాటులో అడగండి అడగడం అడ్డంగా అనుగుణంగా అనుమతించు అనుమతిస్తుంది అయితే ఇప్పటికే ఉన్నారు ఎక్కడైనా ఎప్పుడు ఎవరైనా ఎవరో ఏ ఏదైనా ఏమైనప్పటికి ఒక ఒకరు కనిపిస్తాయి కాదు కూడా గా గురించి చుట్టూ చేయగలిగింది తగిన తర్వాత దాదాపు దూరంగా నిజంగా పై ప్రకారం ప్రక్కన మధ్య మరియు మరొక మళ్ళీ మాత్రమే మెచ్చుకో వద్ద వెంట వేరుగా వ్యతిరేకంగా సంబంధం".split(" ")),e.te.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var t=e.wordcut;t.init(),e.te.tokenizer=function(r){if(!arguments.length||null==r||void 0==r)return[];if(Array.isArray(r))return r.map(function(t){return isLunr2?new e.Token(t.toLowerCase()):t.toLowerCase()});var i=r.toString().toLowerCase().replace(/^\s+/,"");return t.cut(i).split("|")},e.Pipeline.registerFunction(e.te.stemmer,"stemmer-te"),e.Pipeline.registerFunction(e.te.stopWordFilter,"stopWordFilter-te")}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var r="2"==e.version[0];e.th=function(){this.pipeline.reset(),this.pipeline.add(e.th.trimmer),r?this.tokenizer=e.th.tokenizer:(e.tokenizer&&(e.tokenizer=e.th.tokenizer),this.tokenizerFn&&(this.tokenizerFn=e.th.tokenizer))},e.th.wordCharacters="[-]",e.th.trimmer=e.trimmerSupport.generateTrimmer(e.th.wordCharacters),e.Pipeline.registerFunction(e.th.trimmer,"trimmer-th");var t=e.wordcut;t.init(),e.th.tokenizer=function(i){if(!arguments.length||null==i||void 0==i)return[];if(Array.isArray(i))return i.map(function(t){return r?new e.Token(t):t});var n=i.toString().replace(/^\s+/,"");return t.cut(n).split("|")}}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.vi=function(){this.pipeline.reset(),this.pipeline.add(e.vi.stopWordFilter,e.vi.trimmer)},e.vi.wordCharacters="[A-Za-ẓ̀͐́͑̉̃̓ÂâÊêÔôĂ-ăĐ-đƠ-ơƯ-ư]",e.vi.trimmer=e.trimmerSupport.generateTrimmer(e.vi.wordCharacters),e.Pipeline.registerFunction(e.vi.trimmer,"trimmer-vi"),e.vi.stopWordFilter=e.generateStopWordFilter("là cái nhưng mà".split(" "))}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r(require("@node-rs/jieba")):r()(e.lunr)}(this,function(e){return function(r,t){if(void 0===r)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===r.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var i="2"==r.version[0];r.zh=function(){this.pipeline.reset(),this.pipeline.add(r.zh.trimmer,r.zh.stopWordFilter,r.zh.stemmer),i?this.tokenizer=r.zh.tokenizer:(r.tokenizer&&(r.tokenizer=r.zh.tokenizer),this.tokenizerFn&&(this.tokenizerFn=r.zh.tokenizer))},r.zh.tokenizer=function(n){if(!arguments.length||null==n||void 0==n)return[];if(Array.isArray(n))return n.map(function(e){return i?new r.Token(e.toLowerCase()):e.toLowerCase()});t&&e.load(t);var o=n.toString().trim().toLowerCase(),s=[];e.cut(o,!0).forEach(function(e){s=s.concat(e.split(" "))}),s=s.filter(function(e){return!!e});var u=0;return s.map(function(e,t){if(i){var n=o.indexOf(e,u),s={};return s.position=[n,e.length],s.index=t,u=n,new r.Token(e,s)}return e})},r.zh.wordCharacters="\\w一-龥",r.zh.trimmer=r.trimmerSupport.generateTrimmer(r.zh.wordCharacters),r.Pipeline.registerFunction(r.zh.trimmer,"trimmer-zh"),r.zh.stemmer=function(){return function(e){return e}}(),r.Pipeline.registerFunction(r.zh.stemmer,"stemmer-zh"),r.zh.stopWordFilter=r.generateStopWordFilter("的 一 不 在 人 有 是 为 為 以 于 於 上 他 而 后 後 之 来 來 及 了 因 下 可 到 由 这 這 与 與 也 此 但 并 並 个 個 其 已 无 無 小 我 们 們 起 最 再 今 去 好 只 又 或 很 亦 某 把 那 你 乃 它 吧 被 比 别 趁 当 當 从 從 得 打 凡 儿 兒 尔 爾 该 該 各 给 給 跟 和 何 还 還 即 几 幾 既 看 据 據 距 靠 啦 另 么 麽 每 嘛 拿 哪 您 凭 憑 且 却 卻 让 讓 仍 啥 如 若 使 谁 誰 虽 雖 随 隨 同 所 她 哇 嗡 往 些 向 沿 哟 喲 用 咱 则 則 怎 曾 至 致 着 著 诸 諸 自".split(" ")),r.Pipeline.registerFunction(r.zh.stopWordFilter,"stopWordFilter-zh")}});
|
||||
@@ -0,0 +1,206 @@
|
||||
/**
|
||||
* export the module via AMD, CommonJS or as a browser global
|
||||
* Export code from https://github.com/umdjs/umd/blob/master/returnExports.js
|
||||
*/
|
||||
;(function (root, factory) {
|
||||
if (typeof define === 'function' && define.amd) {
|
||||
// AMD. Register as an anonymous module.
|
||||
define(factory)
|
||||
} else if (typeof exports === 'object') {
|
||||
/**
|
||||
* Node. Does not work with strict CommonJS, but
|
||||
* only CommonJS-like environments that support module.exports,
|
||||
* like Node.
|
||||
*/
|
||||
module.exports = factory()
|
||||
} else {
|
||||
// Browser globals (root is window)
|
||||
factory()(root.lunr);
|
||||
}
|
||||
}(this, function () {
|
||||
/**
|
||||
* Just return a value to define the module export.
|
||||
* This example returns an object, but the module
|
||||
* can return a function as the exported value.
|
||||
*/
|
||||
|
||||
return function(lunr) {
|
||||
// TinySegmenter 0.1 -- Super compact Japanese tokenizer in Javascript
|
||||
// (c) 2008 Taku Kudo <taku@chasen.org>
|
||||
// TinySegmenter is freely distributable under the terms of a new BSD licence.
|
||||
// For details, see http://chasen.org/~taku/software/TinySegmenter/LICENCE.txt
|
||||
|
||||
function TinySegmenter() {
|
||||
var patterns = {
|
||||
"[一二三四五六七八九十百千万億兆]":"M",
|
||||
"[一-龠々〆ヵヶ]":"H",
|
||||
"[ぁ-ん]":"I",
|
||||
"[ァ-ヴーア-ン゙ー]":"K",
|
||||
"[a-zA-Za-zA-Z]":"A",
|
||||
"[0-90-9]":"N"
|
||||
}
|
||||
this.chartype_ = [];
|
||||
for (var i in patterns) {
|
||||
var regexp = new RegExp(i);
|
||||
this.chartype_.push([regexp, patterns[i]]);
|
||||
}
|
||||
|
||||
this.BIAS__ = -332
|
||||
this.BC1__ = {"HH":6,"II":2461,"KH":406,"OH":-1378};
|
||||
this.BC2__ = {"AA":-3267,"AI":2744,"AN":-878,"HH":-4070,"HM":-1711,"HN":4012,"HO":3761,"IA":1327,"IH":-1184,"II":-1332,"IK":1721,"IO":5492,"KI":3831,"KK":-8741,"MH":-3132,"MK":3334,"OO":-2920};
|
||||
this.BC3__ = {"HH":996,"HI":626,"HK":-721,"HN":-1307,"HO":-836,"IH":-301,"KK":2762,"MK":1079,"MM":4034,"OA":-1652,"OH":266};
|
||||
this.BP1__ = {"BB":295,"OB":304,"OO":-125,"UB":352};
|
||||
this.BP2__ = {"BO":60,"OO":-1762};
|
||||
this.BQ1__ = {"BHH":1150,"BHM":1521,"BII":-1158,"BIM":886,"BMH":1208,"BNH":449,"BOH":-91,"BOO":-2597,"OHI":451,"OIH":-296,"OKA":1851,"OKH":-1020,"OKK":904,"OOO":2965};
|
||||
this.BQ2__ = {"BHH":118,"BHI":-1159,"BHM":466,"BIH":-919,"BKK":-1720,"BKO":864,"OHH":-1139,"OHM":-181,"OIH":153,"UHI":-1146};
|
||||
this.BQ3__ = {"BHH":-792,"BHI":2664,"BII":-299,"BKI":419,"BMH":937,"BMM":8335,"BNN":998,"BOH":775,"OHH":2174,"OHM":439,"OII":280,"OKH":1798,"OKI":-793,"OKO":-2242,"OMH":-2402,"OOO":11699};
|
||||
this.BQ4__ = {"BHH":-3895,"BIH":3761,"BII":-4654,"BIK":1348,"BKK":-1806,"BMI":-3385,"BOO":-12396,"OAH":926,"OHH":266,"OHK":-2036,"ONN":-973};
|
||||
this.BW1__ = {",と":660,",同":727,"B1あ":1404,"B1同":542,"、と":660,"、同":727,"」と":1682,"あっ":1505,"いう":1743,"いっ":-2055,"いる":672,"うし":-4817,"うん":665,"から":3472,"がら":600,"こう":-790,"こと":2083,"こん":-1262,"さら":-4143,"さん":4573,"した":2641,"して":1104,"すで":-3399,"そこ":1977,"それ":-871,"たち":1122,"ため":601,"った":3463,"つい":-802,"てい":805,"てき":1249,"でき":1127,"です":3445,"では":844,"とい":-4915,"とみ":1922,"どこ":3887,"ない":5713,"なっ":3015,"など":7379,"なん":-1113,"にし":2468,"には":1498,"にも":1671,"に対":-912,"の一":-501,"の中":741,"ませ":2448,"まで":1711,"まま":2600,"まる":-2155,"やむ":-1947,"よっ":-2565,"れた":2369,"れで":-913,"をし":1860,"を見":731,"亡く":-1886,"京都":2558,"取り":-2784,"大き":-2604,"大阪":1497,"平方":-2314,"引き":-1336,"日本":-195,"本当":-2423,"毎日":-2113,"目指":-724,"B1あ":1404,"B1同":542,"」と":1682};
|
||||
this.BW2__ = {"..":-11822,"11":-669,"――":-5730,"−−":-13175,"いう":-1609,"うか":2490,"かし":-1350,"かも":-602,"から":-7194,"かれ":4612,"がい":853,"がら":-3198,"きた":1941,"くな":-1597,"こと":-8392,"この":-4193,"させ":4533,"され":13168,"さん":-3977,"しい":-1819,"しか":-545,"した":5078,"して":972,"しな":939,"その":-3744,"たい":-1253,"たた":-662,"ただ":-3857,"たち":-786,"たと":1224,"たは":-939,"った":4589,"って":1647,"っと":-2094,"てい":6144,"てき":3640,"てく":2551,"ては":-3110,"ても":-3065,"でい":2666,"でき":-1528,"でし":-3828,"です":-4761,"でも":-4203,"とい":1890,"とこ":-1746,"とと":-2279,"との":720,"とみ":5168,"とも":-3941,"ない":-2488,"なが":-1313,"など":-6509,"なの":2614,"なん":3099,"にお":-1615,"にし":2748,"にな":2454,"によ":-7236,"に対":-14943,"に従":-4688,"に関":-11388,"のか":2093,"ので":-7059,"のに":-6041,"のの":-6125,"はい":1073,"はが":-1033,"はず":-2532,"ばれ":1813,"まし":-1316,"まで":-6621,"まれ":5409,"めて":-3153,"もい":2230,"もの":-10713,"らか":-944,"らし":-1611,"らに":-1897,"りし":651,"りま":1620,"れた":4270,"れて":849,"れば":4114,"ろう":6067,"われ":7901,"を通":-11877,"んだ":728,"んな":-4115,"一人":602,"一方":-1375,"一日":970,"一部":-1051,"上が":-4479,"会社":-1116,"出て":2163,"分の":-7758,"同党":970,"同日":-913,"大阪":-2471,"委員":-1250,"少な":-1050,"年度":-8669,"年間":-1626,"府県":-2363,"手権":-1982,"新聞":-4066,"日新":-722,"日本":-7068,"日米":3372,"曜日":-601,"朝鮮":-2355,"本人":-2697,"東京":-1543,"然と":-1384,"社会":-1276,"立て":-990,"第に":-1612,"米国":-4268,"11":-669};
|
||||
this.BW3__ = {"あた":-2194,"あり":719,"ある":3846,"い.":-1185,"い。":-1185,"いい":5308,"いえ":2079,"いく":3029,"いた":2056,"いっ":1883,"いる":5600,"いわ":1527,"うち":1117,"うと":4798,"えと":1454,"か.":2857,"か。":2857,"かけ":-743,"かっ":-4098,"かに":-669,"から":6520,"かり":-2670,"が,":1816,"が、":1816,"がき":-4855,"がけ":-1127,"がっ":-913,"がら":-4977,"がり":-2064,"きた":1645,"けど":1374,"こと":7397,"この":1542,"ころ":-2757,"さい":-714,"さを":976,"し,":1557,"し、":1557,"しい":-3714,"した":3562,"して":1449,"しな":2608,"しま":1200,"す.":-1310,"す。":-1310,"する":6521,"ず,":3426,"ず、":3426,"ずに":841,"そう":428,"た.":8875,"た。":8875,"たい":-594,"たの":812,"たり":-1183,"たる":-853,"だ.":4098,"だ。":4098,"だっ":1004,"った":-4748,"って":300,"てい":6240,"てお":855,"ても":302,"です":1437,"でに":-1482,"では":2295,"とう":-1387,"とし":2266,"との":541,"とも":-3543,"どう":4664,"ない":1796,"なく":-903,"など":2135,"に,":-1021,"に、":-1021,"にし":1771,"にな":1906,"には":2644,"の,":-724,"の、":-724,"の子":-1000,"は,":1337,"は、":1337,"べき":2181,"まし":1113,"ます":6943,"まっ":-1549,"まで":6154,"まれ":-793,"らし":1479,"られ":6820,"るる":3818,"れ,":854,"れ、":854,"れた":1850,"れて":1375,"れば":-3246,"れる":1091,"われ":-605,"んだ":606,"んで":798,"カ月":990,"会議":860,"入り":1232,"大会":2217,"始め":1681,"市":965,"新聞":-5055,"日,":974,"日、":974,"社会":2024,"カ月":990};
|
||||
this.TC1__ = {"AAA":1093,"HHH":1029,"HHM":580,"HII":998,"HOH":-390,"HOM":-331,"IHI":1169,"IOH":-142,"IOI":-1015,"IOM":467,"MMH":187,"OOI":-1832};
|
||||
this.TC2__ = {"HHO":2088,"HII":-1023,"HMM":-1154,"IHI":-1965,"KKH":703,"OII":-2649};
|
||||
this.TC3__ = {"AAA":-294,"HHH":346,"HHI":-341,"HII":-1088,"HIK":731,"HOH":-1486,"IHH":128,"IHI":-3041,"IHO":-1935,"IIH":-825,"IIM":-1035,"IOI":-542,"KHH":-1216,"KKA":491,"KKH":-1217,"KOK":-1009,"MHH":-2694,"MHM":-457,"MHO":123,"MMH":-471,"NNH":-1689,"NNO":662,"OHO":-3393};
|
||||
this.TC4__ = {"HHH":-203,"HHI":1344,"HHK":365,"HHM":-122,"HHN":182,"HHO":669,"HIH":804,"HII":679,"HOH":446,"IHH":695,"IHO":-2324,"IIH":321,"III":1497,"IIO":656,"IOO":54,"KAK":4845,"KKA":3386,"KKK":3065,"MHH":-405,"MHI":201,"MMH":-241,"MMM":661,"MOM":841};
|
||||
this.TQ1__ = {"BHHH":-227,"BHHI":316,"BHIH":-132,"BIHH":60,"BIII":1595,"BNHH":-744,"BOHH":225,"BOOO":-908,"OAKK":482,"OHHH":281,"OHIH":249,"OIHI":200,"OIIH":-68};
|
||||
this.TQ2__ = {"BIHH":-1401,"BIII":-1033,"BKAK":-543,"BOOO":-5591};
|
||||
this.TQ3__ = {"BHHH":478,"BHHM":-1073,"BHIH":222,"BHII":-504,"BIIH":-116,"BIII":-105,"BMHI":-863,"BMHM":-464,"BOMH":620,"OHHH":346,"OHHI":1729,"OHII":997,"OHMH":481,"OIHH":623,"OIIH":1344,"OKAK":2792,"OKHH":587,"OKKA":679,"OOHH":110,"OOII":-685};
|
||||
this.TQ4__ = {"BHHH":-721,"BHHM":-3604,"BHII":-966,"BIIH":-607,"BIII":-2181,"OAAA":-2763,"OAKK":180,"OHHH":-294,"OHHI":2446,"OHHO":480,"OHIH":-1573,"OIHH":1935,"OIHI":-493,"OIIH":626,"OIII":-4007,"OKAK":-8156};
|
||||
this.TW1__ = {"につい":-4681,"東京都":2026};
|
||||
this.TW2__ = {"ある程":-2049,"いった":-1256,"ころが":-2434,"しょう":3873,"その後":-4430,"だって":-1049,"ていた":1833,"として":-4657,"ともに":-4517,"もので":1882,"一気に":-792,"初めて":-1512,"同時に":-8097,"大きな":-1255,"対して":-2721,"社会党":-3216};
|
||||
this.TW3__ = {"いただ":-1734,"してい":1314,"として":-4314,"につい":-5483,"にとっ":-5989,"に当た":-6247,"ので,":-727,"ので、":-727,"のもの":-600,"れから":-3752,"十二月":-2287};
|
||||
this.TW4__ = {"いう.":8576,"いう。":8576,"からな":-2348,"してい":2958,"たが,":1516,"たが、":1516,"ている":1538,"という":1349,"ました":5543,"ません":1097,"ようと":-4258,"よると":5865};
|
||||
this.UC1__ = {"A":484,"K":93,"M":645,"O":-505};
|
||||
this.UC2__ = {"A":819,"H":1059,"I":409,"M":3987,"N":5775,"O":646};
|
||||
this.UC3__ = {"A":-1370,"I":2311};
|
||||
this.UC4__ = {"A":-2643,"H":1809,"I":-1032,"K":-3450,"M":3565,"N":3876,"O":6646};
|
||||
this.UC5__ = {"H":313,"I":-1238,"K":-799,"M":539,"O":-831};
|
||||
this.UC6__ = {"H":-506,"I":-253,"K":87,"M":247,"O":-387};
|
||||
this.UP1__ = {"O":-214};
|
||||
this.UP2__ = {"B":69,"O":935};
|
||||
this.UP3__ = {"B":189};
|
||||
this.UQ1__ = {"BH":21,"BI":-12,"BK":-99,"BN":142,"BO":-56,"OH":-95,"OI":477,"OK":410,"OO":-2422};
|
||||
this.UQ2__ = {"BH":216,"BI":113,"OK":1759};
|
||||
this.UQ3__ = {"BA":-479,"BH":42,"BI":1913,"BK":-7198,"BM":3160,"BN":6427,"BO":14761,"OI":-827,"ON":-3212};
|
||||
this.UW1__ = {",":156,"、":156,"「":-463,"あ":-941,"う":-127,"が":-553,"き":121,"こ":505,"で":-201,"と":-547,"ど":-123,"に":-789,"の":-185,"は":-847,"も":-466,"や":-470,"よ":182,"ら":-292,"り":208,"れ":169,"を":-446,"ん":-137,"・":-135,"主":-402,"京":-268,"区":-912,"午":871,"国":-460,"大":561,"委":729,"市":-411,"日":-141,"理":361,"生":-408,"県":-386,"都":-718,"「":-463,"・":-135};
|
||||
this.UW2__ = {",":-829,"、":-829,"〇":892,"「":-645,"」":3145,"あ":-538,"い":505,"う":134,"お":-502,"か":1454,"が":-856,"く":-412,"こ":1141,"さ":878,"ざ":540,"し":1529,"す":-675,"せ":300,"そ":-1011,"た":188,"だ":1837,"つ":-949,"て":-291,"で":-268,"と":-981,"ど":1273,"な":1063,"に":-1764,"の":130,"は":-409,"ひ":-1273,"べ":1261,"ま":600,"も":-1263,"や":-402,"よ":1639,"り":-579,"る":-694,"れ":571,"を":-2516,"ん":2095,"ア":-587,"カ":306,"キ":568,"ッ":831,"三":-758,"不":-2150,"世":-302,"中":-968,"主":-861,"事":492,"人":-123,"会":978,"保":362,"入":548,"初":-3025,"副":-1566,"北":-3414,"区":-422,"大":-1769,"天":-865,"太":-483,"子":-1519,"学":760,"実":1023,"小":-2009,"市":-813,"年":-1060,"強":1067,"手":-1519,"揺":-1033,"政":1522,"文":-1355,"新":-1682,"日":-1815,"明":-1462,"最":-630,"朝":-1843,"本":-1650,"東":-931,"果":-665,"次":-2378,"民":-180,"気":-1740,"理":752,"発":529,"目":-1584,"相":-242,"県":-1165,"立":-763,"第":810,"米":509,"自":-1353,"行":838,"西":-744,"見":-3874,"調":1010,"議":1198,"込":3041,"開":1758,"間":-1257,"「":-645,"」":3145,"ッ":831,"ア":-587,"カ":306,"キ":568};
|
||||
this.UW3__ = {",":4889,"1":-800,"−":-1723,"、":4889,"々":-2311,"〇":5827,"」":2670,"〓":-3573,"あ":-2696,"い":1006,"う":2342,"え":1983,"お":-4864,"か":-1163,"が":3271,"く":1004,"け":388,"げ":401,"こ":-3552,"ご":-3116,"さ":-1058,"し":-395,"す":584,"せ":3685,"そ":-5228,"た":842,"ち":-521,"っ":-1444,"つ":-1081,"て":6167,"で":2318,"と":1691,"ど":-899,"な":-2788,"に":2745,"の":4056,"は":4555,"ひ":-2171,"ふ":-1798,"へ":1199,"ほ":-5516,"ま":-4384,"み":-120,"め":1205,"も":2323,"や":-788,"よ":-202,"ら":727,"り":649,"る":5905,"れ":2773,"わ":-1207,"を":6620,"ん":-518,"ア":551,"グ":1319,"ス":874,"ッ":-1350,"ト":521,"ム":1109,"ル":1591,"ロ":2201,"ン":278,"・":-3794,"一":-1619,"下":-1759,"世":-2087,"両":3815,"中":653,"主":-758,"予":-1193,"二":974,"人":2742,"今":792,"他":1889,"以":-1368,"低":811,"何":4265,"作":-361,"保":-2439,"元":4858,"党":3593,"全":1574,"公":-3030,"六":755,"共":-1880,"円":5807,"再":3095,"分":457,"初":2475,"別":1129,"前":2286,"副":4437,"力":365,"動":-949,"務":-1872,"化":1327,"北":-1038,"区":4646,"千":-2309,"午":-783,"協":-1006,"口":483,"右":1233,"各":3588,"合":-241,"同":3906,"和":-837,"員":4513,"国":642,"型":1389,"場":1219,"外":-241,"妻":2016,"学":-1356,"安":-423,"実":-1008,"家":1078,"小":-513,"少":-3102,"州":1155,"市":3197,"平":-1804,"年":2416,"広":-1030,"府":1605,"度":1452,"建":-2352,"当":-3885,"得":1905,"思":-1291,"性":1822,"戸":-488,"指":-3973,"政":-2013,"教":-1479,"数":3222,"文":-1489,"新":1764,"日":2099,"旧":5792,"昨":-661,"時":-1248,"曜":-951,"最":-937,"月":4125,"期":360,"李":3094,"村":364,"東":-805,"核":5156,"森":2438,"業":484,"氏":2613,"民":-1694,"決":-1073,"法":1868,"海":-495,"無":979,"物":461,"特":-3850,"生":-273,"用":914,"町":1215,"的":7313,"直":-1835,"省":792,"県":6293,"知":-1528,"私":4231,"税":401,"立":-960,"第":1201,"米":7767,"系":3066,"約":3663,"級":1384,"統":-4229,"総":1163,"線":1255,"者":6457,"能":725,"自":-2869,"英":785,"見":1044,"調":-562,"財":-733,"費":1777,"車":1835,"軍":1375,"込":-1504,"通":-1136,"選":-681,"郎":1026,"郡":4404,"部":1200,"金":2163,"長":421,"開":-1432,"間":1302,"関":-1282,"雨":2009,"電":-1045,"非":2066,"駅":1620,"1":-800,"」":2670,"・":-3794,"ッ":-1350,"ア":551,"グ":1319,"ス":874,"ト":521,"ム":1109,"ル":1591,"ロ":2201,"ン":278};
|
||||
this.UW4__ = {",":3930,".":3508,"―":-4841,"、":3930,"。":3508,"〇":4999,"「":1895,"」":3798,"〓":-5156,"あ":4752,"い":-3435,"う":-640,"え":-2514,"お":2405,"か":530,"が":6006,"き":-4482,"ぎ":-3821,"く":-3788,"け":-4376,"げ":-4734,"こ":2255,"ご":1979,"さ":2864,"し":-843,"じ":-2506,"す":-731,"ず":1251,"せ":181,"そ":4091,"た":5034,"だ":5408,"ち":-3654,"っ":-5882,"つ":-1659,"て":3994,"で":7410,"と":4547,"な":5433,"に":6499,"ぬ":1853,"ね":1413,"の":7396,"は":8578,"ば":1940,"ひ":4249,"び":-4134,"ふ":1345,"へ":6665,"べ":-744,"ほ":1464,"ま":1051,"み":-2082,"む":-882,"め":-5046,"も":4169,"ゃ":-2666,"や":2795,"ょ":-1544,"よ":3351,"ら":-2922,"り":-9726,"る":-14896,"れ":-2613,"ろ":-4570,"わ":-1783,"を":13150,"ん":-2352,"カ":2145,"コ":1789,"セ":1287,"ッ":-724,"ト":-403,"メ":-1635,"ラ":-881,"リ":-541,"ル":-856,"ン":-3637,"・":-4371,"ー":-11870,"一":-2069,"中":2210,"予":782,"事":-190,"井":-1768,"人":1036,"以":544,"会":950,"体":-1286,"作":530,"側":4292,"先":601,"党":-2006,"共":-1212,"内":584,"円":788,"初":1347,"前":1623,"副":3879,"力":-302,"動":-740,"務":-2715,"化":776,"区":4517,"協":1013,"参":1555,"合":-1834,"和":-681,"員":-910,"器":-851,"回":1500,"国":-619,"園":-1200,"地":866,"場":-1410,"塁":-2094,"士":-1413,"多":1067,"大":571,"子":-4802,"学":-1397,"定":-1057,"寺":-809,"小":1910,"屋":-1328,"山":-1500,"島":-2056,"川":-2667,"市":2771,"年":374,"庁":-4556,"後":456,"性":553,"感":916,"所":-1566,"支":856,"改":787,"政":2182,"教":704,"文":522,"方":-856,"日":1798,"時":1829,"最":845,"月":-9066,"木":-485,"来":-442,"校":-360,"業":-1043,"氏":5388,"民":-2716,"気":-910,"沢":-939,"済":-543,"物":-735,"率":672,"球":-1267,"生":-1286,"産":-1101,"田":-2900,"町":1826,"的":2586,"目":922,"省":-3485,"県":2997,"空":-867,"立":-2112,"第":788,"米":2937,"系":786,"約":2171,"経":1146,"統":-1169,"総":940,"線":-994,"署":749,"者":2145,"能":-730,"般":-852,"行":-792,"規":792,"警":-1184,"議":-244,"谷":-1000,"賞":730,"車":-1481,"軍":1158,"輪":-1433,"込":-3370,"近":929,"道":-1291,"選":2596,"郎":-4866,"都":1192,"野":-1100,"銀":-2213,"長":357,"間":-2344,"院":-2297,"際":-2604,"電":-878,"領":-1659,"題":-792,"館":-1984,"首":1749,"高":2120,"「":1895,"」":3798,"・":-4371,"ッ":-724,"ー":-11870,"カ":2145,"コ":1789,"セ":1287,"ト":-403,"メ":-1635,"ラ":-881,"リ":-541,"ル":-856,"ン":-3637};
|
||||
this.UW5__ = {",":465,".":-299,"1":-514,"E2":-32768,"]":-2762,"、":465,"。":-299,"「":363,"あ":1655,"い":331,"う":-503,"え":1199,"お":527,"か":647,"が":-421,"き":1624,"ぎ":1971,"く":312,"げ":-983,"さ":-1537,"し":-1371,"す":-852,"だ":-1186,"ち":1093,"っ":52,"つ":921,"て":-18,"で":-850,"と":-127,"ど":1682,"な":-787,"に":-1224,"の":-635,"は":-578,"べ":1001,"み":502,"め":865,"ゃ":3350,"ょ":854,"り":-208,"る":429,"れ":504,"わ":419,"を":-1264,"ん":327,"イ":241,"ル":451,"ン":-343,"中":-871,"京":722,"会":-1153,"党":-654,"務":3519,"区":-901,"告":848,"員":2104,"大":-1296,"学":-548,"定":1785,"嵐":-1304,"市":-2991,"席":921,"年":1763,"思":872,"所":-814,"挙":1618,"新":-1682,"日":218,"月":-4353,"査":932,"格":1356,"機":-1508,"氏":-1347,"田":240,"町":-3912,"的":-3149,"相":1319,"省":-1052,"県":-4003,"研":-997,"社":-278,"空":-813,"統":1955,"者":-2233,"表":663,"語":-1073,"議":1219,"選":-1018,"郎":-368,"長":786,"間":1191,"題":2368,"館":-689,"1":-514,"E2":-32768,"「":363,"イ":241,"ル":451,"ン":-343};
|
||||
this.UW6__ = {",":227,".":808,"1":-270,"E1":306,"、":227,"。":808,"あ":-307,"う":189,"か":241,"が":-73,"く":-121,"こ":-200,"じ":1782,"す":383,"た":-428,"っ":573,"て":-1014,"で":101,"と":-105,"な":-253,"に":-149,"の":-417,"は":-236,"も":-206,"り":187,"る":-135,"を":195,"ル":-673,"ン":-496,"一":-277,"中":201,"件":-800,"会":624,"前":302,"区":1792,"員":-1212,"委":798,"学":-960,"市":887,"広":-695,"後":535,"業":-697,"相":753,"社":-507,"福":974,"空":-822,"者":1811,"連":463,"郎":1082,"1":-270,"E1":306,"ル":-673,"ン":-496};
|
||||
|
||||
return this;
|
||||
}
|
||||
TinySegmenter.prototype.ctype_ = function(str) {
|
||||
for (var i in this.chartype_) {
|
||||
if (str.match(this.chartype_[i][0])) {
|
||||
return this.chartype_[i][1];
|
||||
}
|
||||
}
|
||||
return "O";
|
||||
}
|
||||
|
||||
TinySegmenter.prototype.ts_ = function(v) {
|
||||
if (v) { return v; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
TinySegmenter.prototype.segment = function(input) {
|
||||
if (input == null || input == undefined || input == "") {
|
||||
return [];
|
||||
}
|
||||
var result = [];
|
||||
var seg = ["B3","B2","B1"];
|
||||
var ctype = ["O","O","O"];
|
||||
var o = input.split("");
|
||||
for (i = 0; i < o.length; ++i) {
|
||||
seg.push(o[i]);
|
||||
ctype.push(this.ctype_(o[i]))
|
||||
}
|
||||
seg.push("E1");
|
||||
seg.push("E2");
|
||||
seg.push("E3");
|
||||
ctype.push("O");
|
||||
ctype.push("O");
|
||||
ctype.push("O");
|
||||
var word = seg[3];
|
||||
var p1 = "U";
|
||||
var p2 = "U";
|
||||
var p3 = "U";
|
||||
for (var i = 4; i < seg.length - 3; ++i) {
|
||||
var score = this.BIAS__;
|
||||
var w1 = seg[i-3];
|
||||
var w2 = seg[i-2];
|
||||
var w3 = seg[i-1];
|
||||
var w4 = seg[i];
|
||||
var w5 = seg[i+1];
|
||||
var w6 = seg[i+2];
|
||||
var c1 = ctype[i-3];
|
||||
var c2 = ctype[i-2];
|
||||
var c3 = ctype[i-1];
|
||||
var c4 = ctype[i];
|
||||
var c5 = ctype[i+1];
|
||||
var c6 = ctype[i+2];
|
||||
score += this.ts_(this.UP1__[p1]);
|
||||
score += this.ts_(this.UP2__[p2]);
|
||||
score += this.ts_(this.UP3__[p3]);
|
||||
score += this.ts_(this.BP1__[p1 + p2]);
|
||||
score += this.ts_(this.BP2__[p2 + p3]);
|
||||
score += this.ts_(this.UW1__[w1]);
|
||||
score += this.ts_(this.UW2__[w2]);
|
||||
score += this.ts_(this.UW3__[w3]);
|
||||
score += this.ts_(this.UW4__[w4]);
|
||||
score += this.ts_(this.UW5__[w5]);
|
||||
score += this.ts_(this.UW6__[w6]);
|
||||
score += this.ts_(this.BW1__[w2 + w3]);
|
||||
score += this.ts_(this.BW2__[w3 + w4]);
|
||||
score += this.ts_(this.BW3__[w4 + w5]);
|
||||
score += this.ts_(this.TW1__[w1 + w2 + w3]);
|
||||
score += this.ts_(this.TW2__[w2 + w3 + w4]);
|
||||
score += this.ts_(this.TW3__[w3 + w4 + w5]);
|
||||
score += this.ts_(this.TW4__[w4 + w5 + w6]);
|
||||
score += this.ts_(this.UC1__[c1]);
|
||||
score += this.ts_(this.UC2__[c2]);
|
||||
score += this.ts_(this.UC3__[c3]);
|
||||
score += this.ts_(this.UC4__[c4]);
|
||||
score += this.ts_(this.UC5__[c5]);
|
||||
score += this.ts_(this.UC6__[c6]);
|
||||
score += this.ts_(this.BC1__[c2 + c3]);
|
||||
score += this.ts_(this.BC2__[c3 + c4]);
|
||||
score += this.ts_(this.BC3__[c4 + c5]);
|
||||
score += this.ts_(this.TC1__[c1 + c2 + c3]);
|
||||
score += this.ts_(this.TC2__[c2 + c3 + c4]);
|
||||
score += this.ts_(this.TC3__[c3 + c4 + c5]);
|
||||
score += this.ts_(this.TC4__[c4 + c5 + c6]);
|
||||
// score += this.ts_(this.TC5__[c4 + c5 + c6]);
|
||||
score += this.ts_(this.UQ1__[p1 + c1]);
|
||||
score += this.ts_(this.UQ2__[p2 + c2]);
|
||||
score += this.ts_(this.UQ3__[p3 + c3]);
|
||||
score += this.ts_(this.BQ1__[p2 + c2 + c3]);
|
||||
score += this.ts_(this.BQ2__[p2 + c3 + c4]);
|
||||
score += this.ts_(this.BQ3__[p3 + c2 + c3]);
|
||||
score += this.ts_(this.BQ4__[p3 + c3 + c4]);
|
||||
score += this.ts_(this.TQ1__[p2 + c1 + c2 + c3]);
|
||||
score += this.ts_(this.TQ2__[p2 + c2 + c3 + c4]);
|
||||
score += this.ts_(this.TQ3__[p3 + c1 + c2 + c3]);
|
||||
score += this.ts_(this.TQ4__[p3 + c2 + c3 + c4]);
|
||||
var p = "O";
|
||||
if (score > 0) {
|
||||
result.push(word);
|
||||
word = "";
|
||||
p = "B";
|
||||
}
|
||||
p1 = p2;
|
||||
p2 = p3;
|
||||
p3 = p;
|
||||
word += seg[i];
|
||||
}
|
||||
result.push(word);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
lunr.TinySegmenter = TinySegmenter;
|
||||
};
|
||||
|
||||
}));
|
||||
|
Before Width: | Height: | Size: 9.5 KiB After Width: | Height: | Size: 9.5 KiB |
|
Before Width: | Height: | Size: 9.8 KiB After Width: | Height: | Size: 9.8 KiB |
|
Before Width: | Height: | Size: 14 KiB After Width: | Height: | Size: 14 KiB |
|
Before Width: | Height: | Size: 87 KiB After Width: | Height: | Size: 87 KiB |
|
Before Width: | Height: | Size: 37 KiB After Width: | Height: | Size: 37 KiB |
|
Before Width: | Height: | Size: 47 KiB After Width: | Height: | Size: 47 KiB |
|
Before Width: | Height: | Size: 88 KiB After Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 124 KiB After Width: | Height: | Size: 124 KiB |
|
Before Width: | Height: | Size: 32 KiB After Width: | Height: | Size: 32 KiB |
|
Before Width: | Height: | Size: 90 KiB After Width: | Height: | Size: 90 KiB |
@@ -1,100 +0,0 @@
|
||||
# Introduction to APIs
|
||||
|
||||
**API**: Application Programming Interface
|
||||
|
||||
## What are web APIs?
|
||||
|
||||
- An API defines the way in which computer systems interact.
|
||||
- We can find APIs in the standard libraries
|
||||
- But a special type of API that is built to be exposed over a network and used remotely, "web APIs".
|
||||
- Those building the API have so much control where as the users have relatively little.
|
||||
- Web APIs allow you to expose *functionality* without exposing the *implementation*.
|
||||
- Sometimes they allow users to take advantage of massive compute.
|
||||
|
||||
## What are resource-oriented APIs?
|
||||
|
||||
- Many web APIs act like servants.
|
||||
- You ask them to do something, and they go off and do it.
|
||||
- This is called *remote procedure call* (**RPC**)
|
||||
|
||||
### So why aren't all APIs RPC-orinented?
|
||||
|
||||
One of the main reasons is the idea of *statefulness*.
|
||||
|
||||
> - **Stateless**: When an API call can be made independently from all other API requests, with no additional context.
|
||||
> - **Statefulness**: A web API that stores context on a user from previous API requests.
|
||||
> For example a web API that stores a user's favourite cities and provides weather forecasts for just those has no runtime inputs but requires a state to be set by the user.
|
||||
|
||||
Consider the following API method names:
|
||||
|
||||
1. `ScheduleFlight()`
|
||||
2. `GetFlightDetails()`
|
||||
3. `ShowAllFlights()`
|
||||
4. `CancelReservation()`
|
||||
5. `RescheduleFlight()`
|
||||
6. `UpgradeTrip()`
|
||||
|
||||
Each one of these RPCs is pretty descriptive, but we have to memorize these methods, each of which is subtly different.
|
||||
|
||||
- e.g. sometimes we talk about flight, other times we talk about a trip or a reservation.
|
||||
- We also need to memorise which action is used in the method.
|
||||
- Was it `ShowFlights()`, `ShowAllFlights()`, `ListFlights()` etc
|
||||
|
||||
We need to standardise, by providing a standard set of building blocks - method-resource
|
||||
|
||||
1. `CreateFlightReservation()`
|
||||
2. `GetFlightReservation()`
|
||||
3. `ListFlightReservation()`
|
||||
4. `DeleteFlightReservation()`
|
||||
5. `UpdateFlightReservation()`
|
||||
|
||||
Resource-oriented APIs will be much easier for users to learn, understand and remember.
|
||||
|
||||
- Standardisation makes it easy to combine what you already know (set of standard actions) which the resource which is easy to learn.
|
||||
|
||||
## What makes an API "good"?
|
||||
|
||||
What is the purpose of building an API in the first place?
|
||||
|
||||
1. We have some functionality that some users want.
|
||||
2. Those users want to use this functionality programmatically
|
||||
|
||||
### Operational
|
||||
|
||||
- The system as a whole must be operational.
|
||||
- It must do the thing users actually want.
|
||||
- **Non-operational** requirements: It must perform how the user expects.
|
||||
- e.g. latency
|
||||
|
||||
### Expressive
|
||||
|
||||
- The system needs to allow users to express the thing they want to do *clearly* and *simply*.
|
||||
- The API should be designed such that there is a clear and simple way to do so.
|
||||
- Avoid workarounds - if there is some functionality a user wants but there is not an easy way to do this, this is called a *workaround*.
|
||||
- e.g. If you have a translation API, users can create a detect language feature by constantly pinging translate endpoint.
|
||||
|
||||
### Simple
|
||||
|
||||
- We could think of simplicity as the number of endpoints.
|
||||
- However an API that relies on a single `ExecuteAction()` method just shifts complexity from one place to another.
|
||||
- APIs should aim to expose the functionality users want in the most straightforward way possible, making the API as simple as possible, but no simpler.
|
||||
- **Make the common case fast**
|
||||
- Whenever you add something that might complicate the API for the benefit of an advanced user, it is best to keep this complexity hidden from a basic user.
|
||||
- This keeps the more frequent scenarios simple and easy whilst enabling advanced features for those who want them.
|
||||
- e.g. Image a translation API. `GET /translate?lang=en`, allowing the user to add a specific language model as a mandatory field is complex for the average user and will slow down basic scenarios.
|
||||
|
||||
### Predictable
|
||||
|
||||
APIs that rely on repeated patterns applied to both the API surface definition and the behaviour.
|
||||
|
||||
Users very rarely learn an entire API, they learn the parts they need to and make assumptions when they need to make additions. e.g. if a query parameter is called text in one endpoint, it should not be called string or query in another.
|
||||
|
||||
APIs that rely on **repeated**, **predictable** patterns are easier and faster to learn; and therefore better.
|
||||
|
||||
### Summary
|
||||
|
||||
- Interfaces are contracts that define how two systems should interact with one another.
|
||||
- APIs are a special type of interface
|
||||
- Web APIs are again a special type of API that is exposed over a network.
|
||||
- **Resource-oriented** APIs are a way of designing APIs to reduce complexity by relying on a standard set of actions, called *methods*, across a limited set of resources.
|
||||
- Good APIs are generally: *operational*, *expressive*, *simple* and *predictable*.
|
||||
@@ -1,31 +0,0 @@
|
||||
# Introduction to API Design Patterns
|
||||
|
||||
## What are API Design Patterns?
|
||||
|
||||
A **software design *pattern*** is a particular design that can be applied over and over to lots of similar software problems, with only minor adjustments. It is not a pre-built library but more of a *blueprint* for solving similarly structured problems.
|
||||
|
||||
- Most often, design patterns focus on specific components rather than entire systems.
|
||||
- e.g. If you want to add a logging system, you can use the **singleton design pattern**.
|
||||
- This pattern is not complete
|
||||
- However, it's well-defined and well-tested pattern to follow when you need to solve this small compartmentalised problem of always having a single instance of a class.
|
||||
|
||||
## Why are API Design Patterns Important?
|
||||
|
||||
- While having programmatic access to a system is very valuable, it's also much more fragile and brittle.
|
||||
- Changes to the interface can easily cause failures for those using the interface.
|
||||
- We refer to this aspect as *flexibility*
|
||||
- Interfaces where users can easily accommodate changes are *flexible*
|
||||
- GUIs are flexible - moving a button
|
||||
- Interfaces where even small changes cause complete failures are *rigid*.
|
||||
- Backend APIs: changing a query parameter breaks old client code.
|
||||
- Rigid interfaces make it much more difficult to iterate toward a great design.
|
||||
- We are often stuck with all design decisions, both good and bad.
|
||||
|
||||
**Pagination Pattern**: The pagination pattern is a way of retrieving a long list of items in smaller, more manageable chunks. The pattern relies on extra fields on both the request and response.
|
||||
|
||||
Moving from a non-paginated to paginated response pattern:
|
||||
|
||||
Q. What happens if we don't start with the pattern?
|
||||
|
||||
1. All previously written clients are expected all the data in one list - it has no way of getting subsequent pages.
|
||||
2. Clients are left to think they have all the data - which can lead to incorrect conclusions.
|
||||
@@ -1,88 +0,0 @@
|
||||
# Naming
|
||||
|
||||
In every software system we build, and every API we design or use - there are names that will live far longer than we ever intend them to. It is **important to choose great names**.
|
||||
|
||||
## Why do names matter?
|
||||
|
||||
When designing and building an API, the names we use will be seen by & interacted with all users of the API.
|
||||
|
||||
## What makes a name "good"?
|
||||
|
||||
### Expressive
|
||||
|
||||
It is critical that a name clearly convey the thing is it naming.
|
||||
|
||||
- e.g. The term topic is used in both messaging and machine learning.
|
||||
- If your project includes both using the name *topic* will be confusing.
|
||||
- A more **expressive** name is required:
|
||||
- `topic_model`
|
||||
- `topic_message`
|
||||
|
||||
### Simple
|
||||
|
||||
- While an expressive name is important, it can also become burdensome if the name is excessively long without adding additional clarity.
|
||||
- Names should be expressive but only to the extent that each additional part of a name adds value to justify its presence.
|
||||
- On the other hand, names shouldn't be oversimplified
|
||||
|
||||
| Name | Note |
|
||||
|------|------|
|
||||
| `UserSpecifiedPreferences` | Expressive, but not simple enough |
|
||||
| `UserPreferences` | Both simple & expressive |
|
||||
| `Preferences` | Too simple |
|
||||
|
||||
### Predictable
|
||||
|
||||
- In general, we should use the same name to represent the same thing, and different names to represent different things.
|
||||
- The basic goal is to allow users of an API to learn one name and continue building on that knowledge to be able to predict what future names would look like.
|
||||
|
||||
## Language, Grammar & Syntax
|
||||
|
||||
Language being inherently flexible and ambiguous can be a good thing and a bad thing.
|
||||
|
||||
- On the one hand, ambiguity allows us to name things to be general enough to support future work.
|
||||
- Naming `image_url` rather than `jpeg_url` presents us from limiting ourselves to a single image format.
|
||||
- One the other hand, when there are multiple ways to express the same thing, we often tend to use them interchangeably, which ultimately makes our naming choices unpredictable.
|
||||
|
||||
### Language
|
||||
|
||||
Use American English.
|
||||
|
||||
### Grammar
|
||||
|
||||
#### Imperative Actions
|
||||
|
||||
REST standard verbs should use the imperative mood. They are all commands or orders.
|
||||
|
||||
- `isValid()`: Should it return simple boolean field? Should it return a list of errors?
|
||||
- `GetValidationErrors()`: Clear that it will return list of errors, empty list if is valid.
|
||||
|
||||
#### Prepositions
|
||||
|
||||
- If a Library API wants to list `Book` resources with the `Author`, it's tempting to name `BooksWithAuthor`.
|
||||
- This falls apart when we add in all our additional resources
|
||||
- We will end up with many function names to call.
|
||||
- The preposition `with` is indicative of a more fundamental problem.
|
||||
- Prepositions act like *code smell*, hinting at something not being quite right.
|
||||
|
||||
#### Pluralisation
|
||||
|
||||
- Most often, we should use the singular.
|
||||
- However collection names might be pluralised.
|
||||
- Use American English to pluralise.
|
||||
|
||||
## Context
|
||||
|
||||
- When we use `book` in the library API, we are referring to the resource, however in a flight booking API - we are referring to an action.
|
||||
|
||||
This means we need to keep the context of our API in mind.
|
||||
|
||||
- Context can impart additional value to a name that might otherwise lack a specific meaning.
|
||||
- It can also lead us astray when we use words with a specific meaning but don't make sense without the context.
|
||||
- *record* is very generic, until you consider the context of an audio recording API.
|
||||
|
||||
## Data types and units
|
||||
|
||||
A name can become more clear when using a richer data type.
|
||||
|
||||
- `dimensions: String;` - this is ambiguous
|
||||
- `dimensions: Dimensions;` (where `Dimensions` is an object)
|
||||
@@ -1,82 +0,0 @@
|
||||
# Resource Scope and Hierarchy
|
||||
|
||||
## What is a resource layout?
|
||||
|
||||
The arrangement of resources in our API, the fields that define those resources, and how those resources relate to one another through those fields.
|
||||
|
||||
In other words, resource layout is the entity (resource) relationship model for a particular design of an API.
|
||||
|
||||
### Types of Relationships
|
||||
|
||||
#### Reference Relationships
|
||||
|
||||
The simplest way or two resources to relate to one another is by a simple reference.
|
||||
|
||||
<figure>
|
||||
<img src="/books/api_design_patterns/media/chapter4_01.png">
|
||||
<figcaption>A message resource contains a reference to a specific user who authored the message.</figcaption>
|
||||
</figure>
|
||||
|
||||
- This reference relationship is sometimes referred to as a *foreign key* relationship.
|
||||
- As a result, this can also be considered a *many-to-one* relationship.
|
||||
- A user might write many messages, but a message always has one user as the author.
|
||||
|
||||
#### Self-Reference Relationships
|
||||
|
||||
<figure>
|
||||
<img src="/books/api_design_patterns/media/chapter4_02.png">
|
||||
<figcaption>An employee resource points at other employee resources as managers and assistants.</figcaption>
|
||||
</figure>
|
||||
|
||||
#### Hierarchical Relationships
|
||||
|
||||
- Hierarchical relationships are sort of like one resource having a pointer to another
|
||||
- But that pointer aims upward and implies more than just one resource pointing at another.
|
||||
- Hierarchies also tend to reflect *containment* or *ownership* between resources.
|
||||
|
||||
<figure>
|
||||
<img src="/books/api_design_patterns/media/chapter4_03.png">
|
||||
<figcaption>ChatRoom resources act as the owner of Message resources through a hierarchical relationship.</figcaption>
|
||||
</figure>
|
||||
|
||||
In this case, there is an implied hierarchy of `ChatRooms` *containing* or *owning* `Messages`.
|
||||
|
||||
## Choosing the Right Relationship
|
||||
|
||||
### Do you need a relationship at all?
|
||||
|
||||
When building an API, after we've chosen the list of things or resources that matter to us, the next step is to decide how these resources relate to one another.
|
||||
|
||||
- Consider a self-reference relationship between `Users`. A single change to one resource can affect millions of other related resources.
|
||||
- e.g. if someone famous deletes their Instagram account, millions of records might be to be removed/updated.
|
||||
|
||||
Reference relationships should be **purposeful** and **fundamental** to the desired behaviour. Any reference relationship should be something important for the API to accomplish its primary goal.
|
||||
|
||||
### References or in-line data
|
||||
|
||||
- Where data is in-lined, we only need a single API call to retrieve all the relevant information.
|
||||
- But what if we aren't interested in that information very often?
|
||||
- Then our response is bloated.
|
||||
|
||||
Optimise for the common case - without compromising the feasibility of the advanced case.
|
||||
|
||||
### Hierarchy
|
||||
|
||||
The biggest differences with this type of relationship are the **cascading** effect of actions and the inheritance of behaviours and properties from parent to child.
|
||||
|
||||
- Deleting a parent resource typically implies deleting a child resource.
|
||||
- Access to a parent generically implies the same level of access to the children resources.
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
### Resources for Everything
|
||||
|
||||
It can often be tempting to create resources for even the tiniest concept you might want to model.
|
||||
|
||||
**Rule of thumb**: If you don't need to interact with one of your resources independent of a resource it's associated with, then it can probably be a data type.
|
||||
|
||||
### Deep Hierarchies
|
||||
|
||||
Overly deep hierarchies can be confusing and difficult to manage.
|
||||
|
||||
Page 63 4.3.3 in-line everything
|
||||
@@ -1,224 +0,0 @@
|
||||
# Chapter 1: Reliable, Scalable and Maintainable Applications
|
||||
|
||||
Many applications today are *data-intensive*, as opposed to *compute-intensive*. Raw CPU power is rarely a limiting factor for these applications.
|
||||
|
||||
A data-intensive application is built from the following building blocks
|
||||
|
||||
- Store data so that they, or another application can find it again later (*databases*)
|
||||
- Remember the result of an expensive operation, to sped up reads (*caches*)
|
||||
- Allow users to search data by keyword or filter it in various ways (*search indexes*)
|
||||
- Send a message to another process, to be handled asynchronously (*stream processing*)
|
||||
- Periodically crunch a large amount of accumulated data (*batch processing*)
|
||||
|
||||
## Thinking about Data Systems
|
||||
|
||||
Database and a message queue are quite similar. They both store data for some time - though they have very different access patterns which means different performance characteristics and thus very different implementations.
|
||||
|
||||
Boundaries between these implementations are becoming slightly blurred. There are data-stores that are also used as message queues (*Redis*) and there are messages queues with database-like durability guarantees (*Apache Kafka*).
|
||||
|
||||
<figure>
|
||||
<img src="/books/designing_data_intensive_applications/media/ddia_0101.gif">
|
||||
<figcaption>One possible architecture for data system that combines several components</figcaption>
|
||||
</figure>
|
||||
|
||||
When you combine several tools in order to provide a service, the service's interface or application programming interface (API) usually hides those implementation details from clients.
|
||||
|
||||
- *Reliability*: The system should continue to work *correctly* (performing the correct function at the desired level of performance) even in the face of *adversity* (hardware or software faults, even human error).
|
||||
- *Scalability*: As the system *grows* (in data volume, traffic volume or complexity), there should be reasonable ways of dealing with that growth.
|
||||
- *Maintainability*: Over time, many different people will work on the system (engineering and operations, both maintaining current behaviour and adapting the system to new use cases), and they should all be able to work in it *productively*.
|
||||
|
||||
## Reliability
|
||||
|
||||
- The application performs the function that the user expected.
|
||||
- It can tolerate the user making mistakes or using the software in unexpected ways.
|
||||
- Its performance is good enough for the required use case, under the expected load and data volume.
|
||||
- The system prevents any unauthorized access and abuse.
|
||||
|
||||
Things that ca go wrong are called *faults*. Systems that anticipate faults and can cope with them are called *fault-tolerant* or *resilient*. Fault tolerance does not mean making a system tolerant of all faults, but only tolerating *certain types* of faults.
|
||||
|
||||
**NOTE**: A fault is not the same as a failure.
|
||||
|
||||
- A fault is defined as one component of the system deviating from its spec.
|
||||
- A failure is when the system as a whole stops providing the required service to the user,
|
||||
|
||||
It is impossible to to reduce the probability of a fault to zero; therefore it is best to design fault-tolerance mechanisms that prevent faults from causing failures.
|
||||
|
||||
### Hardware Faults
|
||||
|
||||
Hard disks are reported as having a mean time to failing (MTTF) of about 10 to 50 years. So on a storage cluster with 10,000 disks, we should expect on average one disk to die per day.
|
||||
|
||||
A good combatant for this is **redundancy**. Disks may be set up in RAID configurations, servers can have dual power supplies etc. When a component dies, the redundant component can take it's place whilst the broken one is being replaced. This approach cannot complete prevent hardware problems from causing failures, but it is well understood and can often keep a machine running uninterrupted for years.
|
||||
|
||||
However, as data volumes and applications' computing demands have increased, more applications have begun using larger number of machines, which proportionally increase the rate of hardware faults. Moreover, in some cloud platforms such as AWS it is fairly common for virtual machine instances to become unavailable without warning as the platforms are designed to prioritise flexibility and elasticity over single-machine reliability.
|
||||
|
||||
Hence there is a move toward systems that can tolerate the loss of entire machines, by using software fault-tolerance techniques in preference or in addition to hardware redundancy. Such systems also have operations advantages: a single-server system requires planned downtime, whereas a system that can tolerate machine failure can be patched one node at a time with no downtime of the entire system (*rolling upgrade*).
|
||||
|
||||
### Software Faults
|
||||
|
||||
Hardware faults are normally random and independent form each other. This is not the case for software faults. Software fault can lie dormant for a long time until they are triggered by am unusual set of circumstances. Though there is no quick solution, there are lots of small ones:
|
||||
|
||||
- Testing
|
||||
- Process isolation
|
||||
- Allowing crash & restart
|
||||
- Measuring and monitoring
|
||||
|
||||
### Human Errors
|
||||
|
||||
Humans design and build software systems, and the operators are also human. Humans are unreliable.
|
||||
|
||||
10%-25% of outages are caused by hardware faults, the rest are human related faults.
|
||||
|
||||
- Design systems in a way that minimises opportunities for error.
|
||||
- e.g. well designed abstractions, APIs and admin interfaces that make it easy to do "the right thing"
|
||||
- Decouple the places where people make the most mistakes from places where they can cause failures
|
||||
- Provide fully featured non-production *sandbox* environments.
|
||||
- Test thoroughly at all levels, from unit tests to whole-system integration tests & manual tests.
|
||||
- Allow quick and easy recovery from human errors to minimise the impact in the case of failure.
|
||||
- Make it fast to roll back configuration changes
|
||||
- Roll out new code gradually
|
||||
- Provide tools to recompute data
|
||||
- Set up detailed and clear monitoring, such as performance metrics and error rates.
|
||||
|
||||
## Scalability
|
||||
|
||||
Even if a system is working reliably today, that doesn't mean it will necessarily work reliably in the future.
|
||||
|
||||
*Scalability* is the term we used to describe a system's ability to cope with increased load.
|
||||
|
||||
### Describing Load
|
||||
|
||||
Load can be described with a few numbers which we call *load parameters*. These parameters depend on the architecture of the system. It might be:
|
||||
|
||||
- Requests per second
|
||||
- Ratio of reads to writes
|
||||
- Number of simultaneous active users
|
||||
- Hit rate on cache
|
||||
|
||||
Consider **Twitter** as an example, they have two main operations, *post tweet* and *home timeline*. There are two ways of implementing these.
|
||||
|
||||
**Approach 1**: Posting a tweet simply inserts the new tweet into a global collection of tweets. When user requests their home timeline, look up all the people they follow, find all the tweets for each of those users and merge them (sorting on time). In a **relational database**
|
||||
```sql
|
||||
SELECT tweets.*, users.*
|
||||
FROM tweets
|
||||
JOIN users ON tweets.sender_id = users.id
|
||||
JOIN follows ON follows.followee_id = users.id
|
||||
WHERE follows.follower_id = current_user
|
||||
```
|
||||
**Approach 2**: Maintain a cache for each user's home timeline - like a mailbox of tweets for each user. When user *posts a tweet*, look up all the people who follow that user, and insert the new tweet into each of their home timeline caches. The request to read the home timeline is the cheap, because its result has been computed ahead of time.
|
||||
|
||||
<figure>
|
||||
<img src="/books/designing_data_intensive_applications/media/ddia_0103.gif">
|
||||
<figcaption>Twitter's data pipeline for delivering tweets to followers, with load parameters</figcaption>
|
||||
</figure>
|
||||
|
||||
The first version of Twitter used approach 1, but the systems struggled to keep up with the load of home timeline queries, so the company switched to approach 2. The average rate of published tweets is almost two orders of magnitude lower than the rate of home timeline reads, so in this case its preferable to do more work at write time and less at read time.
|
||||
|
||||
However the downside of approach 2 is posting a tweet now requires a lot of extra work. On average a tweet is delivered to about 75 followers, so 4.6K tweets/second became 345k writes/second to home timeline caches. However now consider some accounts have 30 million followers.
|
||||
|
||||
Twitter uses a hybrid of both solutions. For users with smaller follow counts approach 2 is used, however for celebrity accounts approach 1 is used and these two timelines are merged together.
|
||||
|
||||
### Describing Performance
|
||||
|
||||
Once you have described the load on your system, you can investigate what happens when load increases.
|
||||
|
||||
- When you increase a load parameter and keep the system resources unchanged, how is the performance of your system affected?
|
||||
- When you increase a load parameter, how much do you need to increase the resources if you want to keep performance unchanged?
|
||||
|
||||
> **LATENCY AND RESPONSE TIME**
|
||||
>
|
||||
> *Latency* and *response time* are often used synonymously, but they are not the same.
|
||||
> **Response time**: Is what the client sees: the sum of service time, network delays and queuing delays.
|
||||
> **Latency**: Is the duration that a request is waiting to be handled - during which it is *latent*, awaiting service.
|
||||
|
||||
<figure>
|
||||
<img src="/books/designing_data_intensive_applications/media/ddia_0104.gif">
|
||||
<figcaption>Illustrating mean and percentiles: response times for a sample of 100 requests to a service</figcaption>
|
||||
</figure>
|
||||
|
||||
Most requests are reasonably fast, but there are occasional *outliers* that take much longer. Perhaps these requests are intrinsically more expensive - however even the same request will see variations due to all matter of reasons.
|
||||
|
||||
*Average* response time of a service is common however it is not a very good metric if you want to know your "typical" response time - it doesn't tell you how many users actually experienced that delay.
|
||||
|
||||
*Percentiles* are a better metric.
|
||||
|
||||
- Take all response times, sort them and the *median* is the half way point.
|
||||
- This makes the median a good metric if you want to know how long users typically have to wait: half of users are served in less than the median, the other half longer. The median is also known as the *50th percentile* and abbreviated as *p50*.
|
||||
- Note this refers to a single request. If a user creates multiple requests, the probability that one of them is over the p50 is much greater than 50%.
|
||||
- In order to figure out how bad your outliers are you can look at higher percentiles: the *95th*, *99th* and *99.9th* (abbreviated to *p95*, *p99* and *p999*).
|
||||
- e.g. if p95 is 1.5 seconds, that means 95 out of 100 requests are served quicker than 1.5 seconds, and 5 are served slower.
|
||||
- High percentiles of response times (also known as *tail latencies*), are important because they directly affect users' experience of the service.
|
||||
|
||||
Amazon descries response time requirements for internal services in terms of p999 even though it only affects 1 in 1000 requests. This is because customers with the slowest requests are often those who have the most data in their accounts (valuable customers).
|
||||
|
||||
Queuing delays often account for a large part of the response time at high percentiles. It only takes a small number of sow requests to hold up the processing of subsequent requests - known as *head-of-line blocking*. Due to this it is important to measure response times on client side.
|
||||
|
||||
<figure>
|
||||
<img src="/books/designing_data_intensive_applications/media/ddia_0105.gif">
|
||||
<figcaption>When several back end calls are needed to serve a request, it takes just a single slow back end request to slow down the entire end-user request.</figcaption>
|
||||
</figure>
|
||||
|
||||
#### Approaches for Coping with Load
|
||||
|
||||
**Vertical Scaling**: Moving to a more powerful machine.
|
||||
|
||||
**Horizontal Scaling**: Distributing the load across multiple smaller machines.
|
||||
|
||||
Some systems are *elastic*, meaning that they can automatically add computing resources when they detect a load increase. Elastic systems are useful if load is unpredictable, but manual/periodic scaled systems are simpler and have fewer operational surprises.
|
||||
|
||||
While distributing stateless services across multiple machines is fairly straightforward, taking stateful data systems from a single node to a distributed set up can introduce additional complexity. Common wisdom (until recently) was to keep your database on a single node and vertically scale until cost dictated horizontal scaling.
|
||||
|
||||
## Maintainability
|
||||
|
||||
Majority of the cost of software is not initial development, but in on going maintenance:
|
||||
|
||||
- Fixing bugs
|
||||
- Keeping systems operational
|
||||
- Investigating failures
|
||||
- Adapting to new platforms
|
||||
- Modifying it for new use cases
|
||||
- Repaying technical debt
|
||||
|
||||
**Operability**: Make it easy for operations teams to keep the system running smoothly.
|
||||
|
||||
**Simplicity**: Make it easy for new engineers to understand the system, by removing as much complexity as possible from the system.
|
||||
|
||||
**Evolvability**: Make it easy for engineers to make changes to the system in the future, adapting it for unanticipated use cases are requirements change. (Also known as *extensibility*, *modifiability* or *plasticity*)
|
||||
|
||||
### Operability: Making Life Easy for Operations
|
||||
|
||||
> "Good operations can work around the limitations of bad software, but good software cannot run reliably with bad operations"
|
||||
|
||||
Operation teams are responsible for the following:
|
||||
|
||||
- Monitoring the health of the system and quickly restoring services.
|
||||
- Tracking down the cause of the problems.
|
||||
- Keeping software and platforms up to date, including security patches.
|
||||
- Keeping tabs on how different systems affect each other.
|
||||
- Anticipating future problems and applying fixes before they occur.
|
||||
- Establishing good practices are tools for deployment and configuration management.
|
||||
- Performing complex maintenance tasks such as moving an application from one platform to another.
|
||||
- Maintaining the security of the system.
|
||||
- Defining processes that make operations predictable and help keep the production environment stable.
|
||||
- Preserving the organisations knowledge about the system, even as individuals come and go.
|
||||
|
||||
Good operability means making routine tasks easy - allowing the operations team to focus their efforts on high-value activities. Data systems can do various things to make routine tasks easy:
|
||||
|
||||
- Providing visibility into the runtime behaviour and internals of the system.
|
||||
- Providing good support for automation and integration with standard tools.
|
||||
- Avoiding dependency on individual machines.
|
||||
- Providing good documentation and easy to understand operational model.
|
||||
- Providing good default behaviour.
|
||||
- Self-healing where appropriate.
|
||||
- Exhibiting predictable behaviour, minimising surprises.
|
||||
|
||||
### Simplicity: Managing Complexity
|
||||
|
||||
In complex software, there is a greater risk of introducing bugs when making a change: when the system is harder for developers to understand and reason about, hidden assumptions, unintended consequences, and unexpected interactions are more easily overlooked.
|
||||
|
||||
Complexity can be *accidental*. This is defined if it is not inherent in the problem the software is trying to solve, but only arises from implementation. One of the best tools for removing accidental complexity is *abstraction*.
|
||||
|
||||
### Evolvability: Making Change Easy
|
||||
|
||||
The ease with which you can modify a data system, and adapt it to changing requirements, is closely linked to its simplicity and its abstractions: simple and easy-to-understand systems are usually easier to modify than complex ones.
|
||||
|
||||
*Evolvability* can be thought of the agility on a data system level.
|
||||
@@ -1,347 +0,0 @@
|
||||
# Chapter 2: Data Models and Query Languages
|
||||
|
||||
Data models are perhaps the most important part of developing software. They define on how we *think about the problem* we are solving.
|
||||
|
||||
Most applications are built by layering one data model on top of another. For each layer the key question is: how is it *represented* in terms of the next-lower layer? For example:
|
||||
|
||||
1. Application developer looks at the real world and model in terms of objects/data structures and APIs that manipulate those data structures.
|
||||
2. Storing is done in JSON, a relational database or a graph model.
|
||||
3. Database engineers then map these structures in terms of bytes in memory on a disk or on a network. This representation needs to allow querying, updating, deletion etc.
|
||||
4. Then the physical layer of actual electrical signals.
|
||||
|
||||
## Relational Model Vs Document Model
|
||||
|
||||
In a relational model, data is organised into *relations* (called *tables* in SQL), where each relation is an unordered collection of *tuples* (*rows* in SQL).
|
||||
|
||||
### The Birth of NoSQL
|
||||
|
||||
\#NoSQL is retroactively interpreted as *Not Only SQL*.
|
||||
|
||||
There are several driving forces behind the adoption of NoSQL databases:
|
||||
|
||||
- A need for greater scalability than relational databases can easily achieve, include very large datasets or very high write throughput.
|
||||
- A widespread preference for free and open source software over commercial database products.
|
||||
- Specialised query operations that are not well supported by the relational model.
|
||||
- Frustration with the restrictiveness of relational schemas, and a desire for a more dynamic and expressive data model.
|
||||
|
||||
### The Object-Relational Mismatch
|
||||
|
||||
Most application development today is done in OOP, meaning if data is stored in relational tables, an awkward transition layer is required between the object in application code and the database model of tables, rows and columns. The disconnect between the models is sometimes called an *impedance mismatch*.
|
||||
|
||||
Object-relational mapping (ORM) frameworks reduce the amount of boiler plate required for this translation layer, but they cannot completely hide it.
|
||||
|
||||
For example, storing a resume on a relational schema can be tricky. The profile as a while can be identified by a unique identifier `user_id`. Fields like `first_name` and `last_name` appear exactly once per user so they can be modeled as columns in the table. However most people have had `n` jobs, this is a one-to-many relationship.
|
||||
|
||||
1. In traditional SQL, jobs would be put in a separate table, with foreign keys in the user table.
|
||||
2. There are some DBs that have added standard support for multi-valued data to be stored in a single row
|
||||
3. Encode this information in a string field as JSON.
|
||||
|
||||
<figure>
|
||||
<img src="/books/designing_data_intensive_applications/media/ddia_0201.jpeg">
|
||||
<figcaption>Representing a LinkedIn profile using a relational schema.</figcaption>
|
||||
</figure>
|
||||
|
||||
Here is the same data stored as a JSON object:
|
||||
|
||||
```json
|
||||
{
|
||||
"user_id": 251,
|
||||
"first_name": "Bill",
|
||||
"last_name": "Gates",
|
||||
"summary": "Co-chair of the Bill & Melinda Gates... Active blogger.",
|
||||
"region_id": "us:91",
|
||||
"industry_id": 131,
|
||||
"photo_url": "/p/7/000/253/05b/308dd6e.jpg",
|
||||
"positions": [
|
||||
{
|
||||
"job_title": "Co-chair",
|
||||
"organization": "Bill & Melinda Gates Foundation"
|
||||
},
|
||||
{
|
||||
"job_title": "Co-founder, Chairman",
|
||||
"organization": "Microsoft"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"school_name": "Harvard University",
|
||||
"start": 1973,
|
||||
"end": 1975
|
||||
},
|
||||
{
|
||||
"school_name": "Lakeside School, Seattle",
|
||||
"start": null,
|
||||
"end": null
|
||||
}
|
||||
],
|
||||
"contact_info": {
|
||||
"blog": "http://thegatesnotes.com",
|
||||
"twitter": "http://twitter.com/BillGates"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The JSON model reduces the impedance mismatch between the application code and the storage layer. The lack of schema is often cited as an advantage.
|
||||
|
||||
The JSON representation has better *locality* than the multi-table schema, if you want to fetch a profile in the relational example, you need to perform multiple queries or a join between 2 or more tables. In the JSON format all relevent data is in one place.
|
||||
|
||||
The one-to-many relationships from the user profile to the user's positions, education, contact information etc imply a tree like structure, the JSON representation makes this tree structure explicit.
|
||||
|
||||
<figure>
|
||||
<img src="/books/designing_data_intensive_applications/media/ddia_0202.gif">
|
||||
<figcaption>One-to-many relationships forming a tree structure</figcaption>
|
||||
</figure>
|
||||
|
||||
### Many-to-One and Many-to-Many Relationships
|
||||
|
||||
In the previous example `region_id` are given as IDs, not as plain-text strings. This is because:
|
||||
|
||||
- Consistent style
|
||||
- Avoids ambiguity (if there are several similarly named cities)
|
||||
- Ease of updating - name is only stored in one place
|
||||
- Localisation support
|
||||
|
||||
Whenever you store an ID or a text string is a question of duplication. When you use an ID, the information that is meaningful to humans is stored in only one place and everything that refers to it uses an ID.
|
||||
|
||||
The advantages of using an ID is that because it has no meaning to humans, it never needs to change: the ID can remain the same, even if the information it identifies changes.
|
||||
|
||||
Anything that is meaningful to humans may need to change sometime in the future - and if that information is duplicated, all the redundant copies need to be updated.
|
||||
|
||||
Removing such duplication is the key idea behind *normalisation* in databases.
|
||||
|
||||
Even if the initial version of an application fits well in a join-free document model, data has a tendency of becoming more interconnected as features are added to applications. See below how adding two extra features turns one-to-many to many-to-many.
|
||||
|
||||
<figure>
|
||||
<img src="/books/designing_data_intensive_applications/media/ddia_0204.gif">
|
||||
<figcaption>Extending resumes with many-to-many relationships</figcaption>
|
||||
</figure>
|
||||
|
||||
### Are Document Databases Repeating History
|
||||
|
||||
While many-to-many relationships and joins are routinely used in relational databases, document databases and NoSQL reopened the debate on how best to represent such relationships in a database.
|
||||
|
||||
This debate is much older than NoSQL - going back to the 1970s.
|
||||
|
||||
#### The Network Model
|
||||
|
||||
In the tree structure of the hierarchical model, every record has exactly one parent; in the network model, a record could have multiple parents.
|
||||
|
||||
For example, there could be one record for the `"Greater Seatlle Area"` region and every user who lived in that region could be linked to it. This allowed one-to-many and many-to-many relationships to be modeled.
|
||||
|
||||
The links between records in the network model were not foreign keys, but more like pointers in a programming language. The only way of accessing a record was to follow a path from a root record along these chains of links. This was called an *access path*.
|
||||
|
||||
In the simplest case, an access path could be like the traversal of a linked list: start at the head of the list and look one record at a time until you find the one you want. But in a world of many-to-many relationships, several different paths can lead to the same record, and a programmer working with the network model had to keep track of these different access paths in their head.
|
||||
|
||||
A **query** was performed by moving a cursor through the database by iterating over lists of records and following access paths. If a record has multiple parents (i.e. multiple incoming pointers from other records), the application code had to keep track of all the various relationships.
|
||||
|
||||
#### The Relational Model
|
||||
|
||||
What the relational model did, by contrast, was to lay out all the data in the open: a relation (table) is simply a collection of tuples (rows), and that it. There are no labyrinthine nested structures, no complicated access paths to follow if you want to query data you can:
|
||||
|
||||
- Read any or all of the rows in a table, selecting those that match your conditions.
|
||||
- Read a particular row by designating some columns as a key and matching on those
|
||||
- Insert a new row into any table without worrying about foreign key relationships to and from other tables.
|
||||
|
||||
The *query optimiser* automatically decides which parts of the query to execute in which order, and which indexes to use.
|
||||
|
||||
Those choices are effectively the equivalent of the "access path", but the big difference is it is made by the query optimiser, not the application developer.
|
||||
|
||||
#### Comparison to Document Databases
|
||||
|
||||
Document databases reverted back to the hierarchical model in one aspect: storing nested records (one-to-many) relationships within their parent record rather than a separate table.
|
||||
|
||||
However, when it come to representing many-to-one and many-to-many relationships, relational and document databases both refer using foreign keys.
|
||||
|
||||
#### Relational Versus Document Databases today
|
||||
|
||||
The main arguments in favour of the document data model are schema flexibility, better performance due to locality, and that for some applications it is closer to the data structures used by the application.
|
||||
|
||||
The relational model counters by providing better support for joins, and many-to-one and many-to-many relationships.
|
||||
|
||||
#### Which data model leads to simpler application code?
|
||||
|
||||
If data in your application has a document-like structure (i.e. a tree of one-to-many relationships where typically the entire tree is loaded at once), then the document model makes sense.
|
||||
|
||||
The relational technique of *shredding* - splitting a document-like structure into multiple tables - can lead to cumbersome schemas and complex code.
|
||||
|
||||
If a document model is deeply nested it can cause problems as nested items cannot be queried directly. For example "the second item in the list of employers for user 251" is inefficient.
|
||||
|
||||
However if you applicaiton does use many-to-many relationships, the document model is less appealing. It's possible to reduce the need for joins by denormalising but then the application code needs to do additional work to keep the denormalised data consistent. Joins can be emulated in application code by making multiple requests to the database. But that moves complexity to the application code and multiple calls is usually slower than the optimised JOIN request.
|
||||
|
||||
#### Schema Flexibility in the Document Model
|
||||
|
||||
No schema means that arbitrary keys can values can be added to a document, and when reading, clients have no guarantees as to what fields the documents may contain.
|
||||
|
||||
Document databases are sometimes called *schemaless*, but that's misleading, as the code that read the data usually assumes some kind of structure. A more accurate term is *schema-on-read*. In contrast *schema-on-write* is enforced by the database on writes.
|
||||
|
||||
For example, say you have currently storing user's full name in one field, however now you want to store them separately. In a document database:
|
||||
|
||||
```js
|
||||
if (user && user.name && !user.first_name) {
|
||||
// Documents written before Dec 8, 2013 don't have first_name
|
||||
user.first_name = user.name.split(" ")[0];
|
||||
}
|
||||
```
|
||||
|
||||
On the other hand, in a "statically typed" database *schema-on-write* approach.
|
||||
|
||||
```sql
|
||||
ALTER TABLE users
|
||||
ADD COLUMN first_name text;
|
||||
UPDATE users
|
||||
SET first_name = split_part(name, ' ', 1);
|
||||
```
|
||||
|
||||
Altering the table is relatively quick however setting every row in the table is time consuming.
|
||||
|
||||
The schema-on-read approach is advantageous if the items in the collection don't all have the same structure.
|
||||
|
||||
#### Data Locality for Queries
|
||||
|
||||
A document is usually stored as a single continuous string, encoded as JSON or binary (MongoDB's BSON). If your application often needs access to the entire document (e.g. rendering to a web page), there is a performance advantage to this *storage locality*. If data is split across multiple tables, multiple index lookups are required to retrieve it all.
|
||||
|
||||
The database typically needs to load the entire document, even if you access only a small portion of it. On updates to a document, the entire document usually needs to be rewritten - only modifications that don't change encoded size can be performed in place (rare).
|
||||
|
||||
For this reason its recommended to keep documents small and avoid frequent updates.
|
||||
|
||||
Some relational databases can offer this locality. Oracle's feature: *multi-table index cluster tables* which declares rows should be inter-leaved in the parent table. There is also the *column-family* concept in Cassandra.
|
||||
|
||||
#### Convergence of document and relational databases
|
||||
|
||||
Relational databases have supported XML since their inception - however many now support JSON.
|
||||
|
||||
Document databases now supports relational like joins in its query language and some MongoDB drivers automatically resolve database references.
|
||||
|
||||
It seems that relational and document databases are becoming more similar over time, and that is a good thing: the data models complement each other. If a database is able to handle document-like data and also perform relational queries on it, applications can use the combination of features that best fits their needs.
|
||||
|
||||
### Query Languages for Data
|
||||
|
||||
**SQL** is a *declarative* query language.
|
||||
|
||||
*Imperative* example:
|
||||
```js
|
||||
function getSharks() {
|
||||
var sharks = [];
|
||||
for(var i = 0; i < animals.length; i++) {
|
||||
if (animals[i].family === "Sharks") {
|
||||
sharks.push(animals[i]);
|
||||
}
|
||||
return sharks;
|
||||
}
|
||||
```
|
||||
In relational algebra, you would instead write:
|
||||
$$
|
||||
sharks = \sigma_{family =''Sharks''} (animals)
|
||||
$$
|
||||
|
||||
Where $\sigma$ is the selection operator, returning only those animals that match the condition $family = ''Sharks''$. SQL follows this closely.
|
||||
|
||||
```SQL
|
||||
SELECT * FROM animals WHERE family = 'Sharks';
|
||||
```
|
||||
|
||||
An imperative language tells the computer to perform certain operations in a certain order.
|
||||
|
||||
In a declarative query language, you just specify the pattern of the data you want. e.g. what conditions should be met, how the data should be transformed - but not *how* to achieve that goal. The declarative query language hides the implementation details of the database engine. This allows the database engine to be optimised and improved without the need to change the query language itself.
|
||||
|
||||
Declarative languages are very easy to parallelise - they specify the pattern of results not the algorithm to be used.
|
||||
|
||||
#### Declarative Queries on the Web
|
||||
|
||||
```html
|
||||
<ul>
|
||||
<li class="selected"><p>Sharks</p></li>
|
||||
<li><p>Whales</p></li>
|
||||
<li><p>Fish</p></li>
|
||||
</ul>
|
||||
```
|
||||
|
||||
```css
|
||||
li.selected > p {
|
||||
background-color: blue;
|
||||
}
|
||||
```
|
||||
|
||||
Here the CSS selector `li.selected > p` declares the pattern of elements to colour blue: all `<p>` elements whise direct parent is a `<li>` element which a class of `selected`.
|
||||
|
||||
Doing this with an imperative approach is a nightmare.
|
||||
```js
|
||||
const liElements = document.getElementsByTagName("li");
|
||||
const selectedLiElements = liElements.filter(liElement => liElement.className === "Selected")
|
||||
for (selectedElement : selectedLiElements) {
|
||||
for (child : selectedElement.childrenNodes()) {
|
||||
if (child.tagName === "p") {
|
||||
child.setAttribute("style", "background-color: blue")
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- If the *selected* class is removed because the user clicks onto a different page, the colour won't be removed - even if the code is re-run, so the item will remain highlighted until refresh. With CSS the browser automatically detects when the rule no longer applies.
|
||||
- If you want to take advantage of a new API, such as `document.getElementsByClassName()`, the code will have to be entirely re-written. On the other hand browsers can improve the performance of CSS without breaking compatibility.
|
||||
|
||||
#### MapReduce Querying
|
||||
|
||||
*MapReduce* is a programming model for processing large amount of data in bulk across many machines. This is supported by MongoDB as a mechanism for performing read-only queries across many documents.
|
||||
|
||||
MapReduce is neither declarative nor imperative but somewhere in between.
|
||||
|
||||
Example in PostgreSQL
|
||||
```SQL
|
||||
SELECT date_trunc('month', observation_timestamp) as observation_month, sum(num_animals) AS total_animals
|
||||
FROM observations
|
||||
WHERE family = "Sharks"
|
||||
GROUP BY observation_month;
|
||||
```
|
||||
|
||||
Example in MongoDB using MapReduce
|
||||
```
|
||||
db.observations.mapReduce(
|
||||
function map() {
|
||||
var year = this.observationTimestamp.getYear();
|
||||
var month = this.observationTimestamp.getMonth();
|
||||
|
||||
return [`${year}-${month}`, this.numAnimals];
|
||||
},
|
||||
function reduce(key, values) {
|
||||
return Array.sum(values);
|
||||
},
|
||||
query: {
|
||||
family: "Sharks"
|
||||
},
|
||||
out: {
|
||||
"monthlySharkReport"
|
||||
}
|
||||
);
|
||||
```
|
||||
|
||||
The `map` function would be called once for each document (e.g. returning `["2026-01", 3], ["2026-01", 4]`. Subsequently the `reduce` function would be called `["2026-01", [3,4]]` returning 7.
|
||||
|
||||
Map and Reduce functions must be pure with no side effects (no additional db calls). This allows them to be run anywhere, in any order and re-run on failure.
|
||||
|
||||
MapReduce was replaced by the *aggregation pipeline*.
|
||||
|
||||
```json
|
||||
{
|
||||
"$match": {
|
||||
"family": "Sharks"
|
||||
}
|
||||
},
|
||||
{
|
||||
"$group": {
|
||||
"_id": {
|
||||
"year": {
|
||||
"$year": "$observationTimestamp"
|
||||
},
|
||||
"month": {
|
||||
"$month": "$observationTimestamp"
|
||||
}
|
||||
},
|
||||
"totalAnimals": {
|
||||
"$sum": "$numAnimals"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Aggregation pipeline language is similar in expressiveness to a subset of SQL, but it uses JSON syntax rather than SQL's English sentence style.
|
||||
@@ -1,18 +0,0 @@
|
||||
# Preface
|
||||
|
||||
There many been many developments in distributed systems, databases and the applications build on top of them, there are various driving forces:
|
||||
|
||||
1. Handling huge volumes of data.
|
||||
2. Businesses need to be agile, test hypotheses cheaply and respond quickly to markets.
|
||||
3. Free & open source software has become very successful and is preferred now to commercial or in-house solutions
|
||||
4. CPU clock speeds are barely increasing. But multi-core processors are standard and networks are getting faster. Parallelism is only going to increase.
|
||||
5. Even small teams can now build systems that are distributed across machines and regions - thanks to **IaaS** (think *AWS*)
|
||||
6. Many services are expected to be highly available. Extended downtime is unacceptable.
|
||||
|
||||
An application is *data-intensive* if data is it's primary challenge.
|
||||
|
||||
- The quantity of data.
|
||||
- The complexity of data.
|
||||
- The speed at which data changes.
|
||||
|
||||
This is opposed to *compute-intensive* where the CPU is the bottle neck.
|
||||
@@ -1 +0,0 @@
|
||||
# Umbra Notes
|
||||
@@ -1,79 +0,0 @@
|
||||
### Module contents
|
||||
|
||||
##### Part 1
|
||||
|
||||
* Mobile Ad Hoc Networks (MANETs)
|
||||
* Delay/Disconnection Tolerant Networks (DTNs)
|
||||
* Vehicular Ad Hoc Networks (VANETs)
|
||||
|
||||
##### Part 2
|
||||
|
||||
* Network experimentation, criteria and evaluations. This part is to help with coursework
|
||||
|
||||
##### Part 3
|
||||
|
||||
* Peer to Peer (P2P)
|
||||
* Content Centric Networks (CCNs)
|
||||
* Information Centric Networks (ICNs)
|
||||
|
||||
##### Part 4
|
||||
|
||||
* Software Defined Networks (SDNs) and Applications
|
||||
|
||||
# Mobile Social Networks
|
||||
|
||||
They have two parts: physical part and a social part
|
||||
|
||||
Social structures are vital for these networks - think covid tracking networks
|
||||
|
||||

|
||||
|
||||
Clouds have multiple layers
|
||||
|
||||
* Network interfaces
|
||||
* Request & accept sensor data
|
||||
* resource management
|
||||
* communicate with other clouds
|
||||
* Processing layer
|
||||
* Store raw data
|
||||
* filter noise
|
||||
* Analysing layer
|
||||
* produce trend chart
|
||||
* learn & predict user behaviour
|
||||
* Services
|
||||
* Interactive dashboard
|
||||
* notification service
|
||||
* sharing access
|
||||
|
||||
|
||||
|
||||
## Vehicle Ad Hoc Networks
|
||||
|
||||
Have social characteristics as they are driven by humans
|
||||
|
||||
VANETs may refer to robots or drones.
|
||||
|
||||
This can be used to exchange warning and beacon messages via V2V (vehicle to vehicle) as well as V2I (vehicle to infrastructure) channels.
|
||||
|
||||
### Fully autonomous Vehicles
|
||||
|
||||

|
||||
|
||||
Vehicles can connect to the cloud and share & request information to help other vehicles.
|
||||
|
||||

|
||||
|
||||
An example of transient clouds - in this case vehicular clouds.
|
||||
|
||||
This can be useful for informing cars behind about congestion. This is real time communication (order of ms which is needed for when cars are moving at 70 mph), cloud communication is not fast enough, due to the data needing to be processed before shared.
|
||||
|
||||
## Challenges
|
||||
|
||||
* Optimal forwarding/routing
|
||||
* Congestion avoidance and control
|
||||
* Security and privacy aware communications
|
||||
* black & grey hole attacks
|
||||
* Energy efficient communications
|
||||
* Important for mobile devices & electric cars
|
||||
* Service provision
|
||||
* Location based services
|
||||
@@ -1,70 +0,0 @@
|
||||
# Mobile Ad Hoc Networks (MANETs)
|
||||
|
||||
* An infrastructure-less network formed by mobile wireless nodes
|
||||
* Nodes in MANET can communicate via single or multi-hop approach (due to absence of centralised network infrastructure)
|
||||
* Nodes operate as clients, routers and servers at the same time to forward packets
|
||||
* The mobility of nodes results in frequent and unpredictable changes in network topology
|
||||
|
||||
One of the core features of a MANET node is the ability to autonomously connect to other nodes and configure itself for data transmission over the network.
|
||||
|
||||
|
||||
#### MANET Routing
|
||||
|
||||
* Mobile wireless nodes create a temporary connection between them to forward data
|
||||
* Because some nodes may not be cooperative or faulty, they may drop/compromise packets
|
||||
* Typically routing is split into **route discovery** and **actual data transmission**.
|
||||
* Nodes have to self organise in order to route.
|
||||
|
||||

|
||||
|
||||
(green boxes is route chosen)
|
||||
|
||||
The source has a limited range of nodes it can detect, it cannot send it direct to the destination as it doesn't know where the destination is. Hops are decided by communication protocols.
|
||||
|
||||
#### Proactive MANETs
|
||||
|
||||
* Also known as table driven routing protocol
|
||||
* Nodes in the network maintain a comprehensive routing information of the network
|
||||
* This is done by spreading network status information to nodes and tracking changes in network topology - think the network is constantly pinged
|
||||
* These status updates can slow the network with the traffic
|
||||
* Useful if the network is not that large
|
||||
|
||||
#### Reactive MANETs
|
||||
|
||||
* Also known as on-demand routing
|
||||
* Network nodes only store information of paths to destination nodes
|
||||
* Nodes delay the search for routes to new destinations in order to reduce communication overheads
|
||||
* i.e. if a route is found between A and B, this route will be stored and not recalculated
|
||||
* May be slower, as a shorter path may not be used
|
||||
|
||||
#### Hybrid MANETs
|
||||
|
||||
* Hybrid protocols combine the advantages of proactive and reactive protocols to reduce traffic overheads and route discovery delays
|
||||
|
||||
Table showing all different protocols of MANETs
|
||||
|
||||

|
||||
|
||||
### Delay/Disconnection Tolerance
|
||||
|
||||
Traditional MANET routing protocols like DSR and AODV (both reactive) cannot work in intermittent infrastructure-less environments because they require a complete path from source to destination for communication.
|
||||
|
||||
* Messages get dropped at intermediate nodes when the link to the next hop is none existent in MANETs
|
||||
* DTNs expand MANETs to allow more intermittent and sparse connections of nodes caused by node mobility or low transmission range.
|
||||
|
||||
#### Store-carry-forward Paradigm
|
||||
|
||||
* DTN routing protocols allow forwarding of messages by using a 'store-carry-forward' approach.
|
||||
* messages are stored by nodes and moved in hops throughout the network until messages reach their destination
|
||||
* This approach is used by DTN routing protocols to increase the probability of message delivery.
|
||||
|
||||
#### DTN Protocol Classifications
|
||||
|
||||
##### Flooding based
|
||||
|
||||
* Flooding based routing protocols spread a message and have multiple copies of the message in the network.
|
||||
* This is done to increase the probability of messages reaching their destination and also decrease the time of delivery
|
||||
|
||||
##### Forwarding based
|
||||
|
||||
* Forwarding based routing protocols gather information about the nodes in a network to select the best path to forward messages with the aim of enhancing message delivery networks with limited resources.
|
||||
@@ -1,102 +0,0 @@
|
||||
# Vehicular Ad Hoc Networks
|
||||
|
||||
* VANETs are a special type of Mobile Ad Hoc network which is used to
|
||||
* provide communication between vehicles that are nearby (V2V)
|
||||
* between vehicles on the road and fixed infrastructures on the roadside (V2I)
|
||||
* VANETs provide complementary approach for intelligent transport system (ITS) and are characterised by **high node mobility** and the limited degree of freedom in the mobility patterns.
|
||||
|
||||
##### Categories of information
|
||||
|
||||
1. Safety application information
|
||||
* e.g. information regarding an accident that has just occurred
|
||||
* the current conditions of the road
|
||||
2. Convenience application
|
||||
* traffic information
|
||||
* parking availability
|
||||
3. Commercial application for pleasure
|
||||
* games
|
||||
* real-time video relay
|
||||
|
||||
### Why do VANETs need different protocols to MANETs
|
||||
|
||||
###### Large scale
|
||||
|
||||
> All vehicles on the road are potential nodes in the VANET.
|
||||
|
||||
###### Predictive Mobility
|
||||
|
||||
> The nodes in a VANET cannot follow arbitrary direction, they have to stay on the road and cannot suddenly change their direction.
|
||||
|
||||
###### High Mobility
|
||||
|
||||
> The network mobility in a VANET changes rapidly due to vehicular speeds.
|
||||
|
||||
###### Partitioned Network
|
||||
|
||||
> The ranges of wireless communication used in V2V networks is near 1 km but vehicles can get disconnected. Can be thought of many disconnected networks.
|
||||
|
||||
The nodes in the VANET can move at **high speeds** which **reduces transmission capacity**, this causes the following issues:
|
||||
|
||||
* **Rapid changes in the network topology** because the state of connectivity between nodes is dynamically changing.
|
||||
|
||||
* **Occasional disconnections due to low traffic density**. This keeps the nodes distant from each other and results to **link failure** that could last for awhile.
|
||||
|
||||
* **Node congestion**, a high traffic situation which affects protocol performance.
|
||||
|
||||
### WAVE IEEE 802.11p
|
||||
|
||||
WAVE - Wireless Access for Vehicular Environment
|
||||
|
||||
* In WAVE vehicles communicate in a **hop by hop** manner with each other
|
||||
* The area of coverage for the WAVE node is limited to 300m-800m
|
||||
* Beyond this range cars cannot communicate
|
||||
|
||||
If there is dense traffic in the coverage region, **nodes become easily congested** because all nodes will be transmitting the same message to every other node.
|
||||
|
||||
> To overcome the limitation of restricted coverage region, the use of DTNs was implemented which uses a **store-carry-forward paradigm**.
|
||||
>
|
||||
> With the store-carry-forward approach, a vehicle stores a message in a buffer and carries the message with it. When it comes into contact with another node, it forwards the message.
|
||||
>
|
||||
> * This introduced the idea of the **Vehicular Delay Tolerant Network (VDTN)** concept
|
||||
|
||||
#### Vehicular Delay Tolerant Network (VDTN)
|
||||
|
||||
VDTNs enable communication in the face of connectivity issues such as
|
||||
|
||||
* long and variable delay
|
||||
* sparse and intermittent connectivity
|
||||
* high error rates
|
||||
* high latency
|
||||
* high asymmetric data rate
|
||||
|
||||
Communication is made possible in the network when intermediate nodes become **custodians** of the message being transmitted and then forward the message only when a opportunity arises.
|
||||
|
||||
###### Fixed DTN nodes
|
||||
|
||||
* The stationary or relay nodes have store and forward capabilities and are located at **road-side intersections** (road side units)
|
||||
* They allow mobile nodes that pass by to collect and leave data on them.
|
||||
* They contribute to increasing the frequency of node contacts and improve **delivery ratio** and **delivery delay**.
|
||||
|
||||

|
||||
|
||||
## Categories of VANETs
|
||||
|
||||
##### Pure cellular/WLAN
|
||||
|
||||
> Pure cellular VANETs may use **fixed cellular gateways and WiMAX access points at road** intersections to gather information
|
||||
>
|
||||
> * note these road side gateways may not be feasible due to cost of infrastructure
|
||||
>
|
||||
> The information collected from sensors of a vehicle in the VANET can become valuable in notifying other nodes about the situation of the traffic in the network.
|
||||
|
||||
##### Pure Ad-Hoc
|
||||
|
||||
> Pure Ad-Hoc architecture is **not reliant** on infrastructure nodes
|
||||
>
|
||||
> In this architecture, nodes perform vehicle to vehicle (V2V) communication with each other.
|
||||
|
||||
##### Hybrid
|
||||
|
||||
> The hybrid category is a combination of the first two. It provides a richer content and offers great **flexibility in the sharing of data**
|
||||
>
|
||||
> * Some vehicles with WLAN and cellular capabilities may be used as **gateways** and **mobile routers** so that vehicles with only WLAN capabilities can interact and communicate effectively with them via multi-hop links.
|
||||
@@ -1,65 +0,0 @@
|
||||
# DTN Protocols
|
||||
|
||||
### Forwarding Based
|
||||
|
||||
Where each message may only be under the custody of a single node.
|
||||
|
||||
* Upon forwarding the message, the receiving node also takes on the responsibility of custody.
|
||||
* This means there will exist only one copy of the message within the network at any period of time.
|
||||
|
||||
#### Direct Transmission
|
||||
|
||||
* Direct transmission is the simplest single-copy forwarding protocol possible.
|
||||
* Once the source has generated a message, it will retain custody and carry it until it encounters the destination.
|
||||
* Once a connection with the destination is established, the message is forwarded directly
|
||||
* This uses minimal resources
|
||||
* Has unbounded amounts of latency
|
||||
* Probability of a message being delivered is only as likely as the probability of the node encountering the destination node
|
||||
|
||||
#### First Contact
|
||||
|
||||
* First contact is a single-copy based forwarding protocol - it randomly chooses a node out of all possible nodes and forwards as many messages as possible to that node.
|
||||
* If no connections are available, the first encountered node will be used.
|
||||
* Once the message(s) are sent, the messages on the original node are deleted, relinquishing custody to the new node.
|
||||
* This protocol routes messages throughout the network via a random walk pattern.
|
||||
* This can lead to packets being routed to dead ends.
|
||||
* Packets can make negative progress or getting stuck in a loop.
|
||||
|
||||
### Replication Based
|
||||
|
||||
Replication-based protocols disseminate messages throughout the network via replication of the messages.
|
||||
|
||||
* When one node encounters another, it will forward the message while retaining the local copy it has.
|
||||
* The existence of multiple copies increases the probability of message delivery and reduces latency.
|
||||
* The more nodes carrying the message, the more chance one node encounters the destination.
|
||||
* However this also means there are many redundant messages on the network - therefore more resources are needed.
|
||||
|
||||
#### Epidemic
|
||||
|
||||
* Utilising the flooding concept, Epidemic aims to achieve message delivery by flooding the network with message copies.
|
||||
* When any two nodes meet, they compare messages.
|
||||
* They then exchange messages they do not have in common
|
||||
* This is repeated allowing the messages to spread similar to an epidemic.
|
||||
* This method achieves minimal latency & high delivery probabilities however suffers from limited resources.
|
||||
|
||||
#### MaxProp
|
||||
|
||||
* Like epidemic, maxprop floods the network, however each message has a priority.
|
||||
* Messages stored in a **ordered-queue** in the **message buffer**.
|
||||
* Messages with a higher probability of being delivered have a higher priory of being forwarded first.
|
||||
* To determine the probability, it looks at **history of encounters**, maintaining a vector with **tracks the likelihood of the node encountering any other node in the network**.
|
||||
* When two nodes meet, they exchange messages and vectors, updating their own local copy.
|
||||
* These vectors are then used to compute the shortest path for each message, messages are then ordered within the buffer by destination cost.
|
||||
* MaxProp uses overhead messages to acknowledge when a message has reached it destination
|
||||
* Once this ACK signal is received, all local copies of redundant messages are dropped.
|
||||
|
||||
#### PROPHET
|
||||
|
||||
Probabilistic Routing Protocol using History of Encounters and Transitivity (PRoPHET)
|
||||
|
||||
* PROPHET maintains a vector that keeps track of a history of the encountered nodes.
|
||||
* It uses this vector to calculate the probability of a message copy reaching its destination by being forwarded to a particular node.
|
||||
* When a source node forwards a message copy, it selects a subset of nodes that it can possibly send to.
|
||||
* The algorithm then **ranks these nodes** based on the calculated probabilities, with the copy being forwarded to the highest ranked nodes first.
|
||||
* This is effective however the routing tables **rapidly grow** as a result of the amount of information on the nodes required to calculate the probability predictions.
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
# DTN Protocols - Advanced
|
||||
|
||||
**Rate control** - addresses easing network congestion by controlling the rate of traffic on the network.
|
||||
|
||||
**Adaptive forwarding** - to direct traffic away from congestion hot spots.
|
||||
|
||||
#### Spray and Focus
|
||||
|
||||
* Spray and focus replicates an allowable number of messages from source in the spray phase.
|
||||
* **The focus phase allows** each node to forward a copy of its messages to other potential nodes until the messages gets to its destination.
|
||||
* The protocol uses a single-copy utility based routing scheme to forward a copy of the message further.
|
||||
* Forwarding decisions are made based on **timers** which record the times nodes come in communication range of each other.
|
||||
* Node $$A$$ forwards message with destination $$D$$ to node $$B$$ , **if and only if** $$B$$ has a higher potential of delivering the message to $$D$$.
|
||||
|
||||
#### SimBet
|
||||
|
||||
* A source node with no prior knowledge of the destination node will forward a message to a more central node that has the potential of finding a suitable relay node.
|
||||
* A central node has the ease of connecting other nodes in a network.
|
||||
* This is known as **centrality** a measure of the **structural importance** of a node in a network.
|
||||
* A central node uses **similarity and betweenness centrality** to avoid unnecessary information exchange in the entire network.
|
||||
* SimBet maintains a single copy of each message in the network to reduce resource overheads.
|
||||
|
||||
### Replication Management
|
||||
|
||||
Replication Management refers to easing network congestion by managing the amount and frequency that messages are replicated.
|
||||
|
||||
* This is particularly notable concern as it is often the replication of messages that leads to congestion in DTNs, with surplus and redundant messages causing wastage within node message buffers.
|
||||
|
||||
#### Café
|
||||
|
||||
* Congestion Aware Forwarding Algorithm (Café)
|
||||
* Single-copy
|
||||
* Adaptive forwarding techniques - to reduce network congestion by directing traffic away from nodes experiencing congestion to less congested areas of the network.
|
||||
* Uses **Contact Manager** and **Congestion Manager**
|
||||
|
||||
**Contact Manager** - deals with nodes forwarding heuristics, updating statistics for each contact such as frequency and duration's.
|
||||
|
||||
**Congestion Manager** - focuses on calculating the availability of nodes, keeping and updating a record of information such as the amount of available buffer and delays expected from each contacted node.
|
||||
|
||||
#### CafREP
|
||||
|
||||
* Congestion Aware Forwarding and Replication (CafREP)
|
||||
* replication-based
|
||||
* builds on Cafe protocol by coalescing the proposed **adaptive forwarding algorithm with an adaptive replication management technique**
|
||||
@@ -1,96 +0,0 @@
|
||||
# Framework for Congestion Control in Delay Tolerant Opportunistic Networks
|
||||
|
||||
DTNs mainly focus on increasing the probability to deliver to the destination and on minimising delays
|
||||
|
||||
* Using complex graph theory techniques
|
||||
* Where load is unfairly distributed towards the better connected nodes
|
||||
* May lead to network congestion
|
||||
|
||||
## CAFREP
|
||||
|
||||
CAFREP or Congestion Aware Forwarding and Replication
|
||||
|
||||
* Detects the congested nodes and parts of the network
|
||||
* Moves the traffic away from hot-spots and spreads it around while preserving the directionality of the traffic and not overwhelming non-interested nodes with unwanted content
|
||||
* Adaptively change message replication rates
|
||||
|
||||
When deciding on the best carrier and the optimal number of messages, CAFREP dynamically combines three heuristics
|
||||
|
||||
1. **Contact** analytics
|
||||
2. Predictive **node congestion** (node storage and in-network delays)
|
||||
3. Predictive **ego network congestion**
|
||||
|
||||

|
||||
|
||||
Each layer you go up, the more information is exchanged between the nodes.
|
||||
|
||||
### Metrics
|
||||
|
||||
###### Node Retentiveness
|
||||
|
||||
- Aims to avoid or replicate proportionally less at the **nodes** that have lower buffer availability.
|
||||
|
||||
$$
|
||||
Ret(X) = B_c(X) - \sum^N_{i=1} \space M^i_{size}(X)
|
||||
$$
|
||||
|
||||
For a node $$X$$, it has buffer of size $$B_c(X)$$. When a message of size $$M^i_{size}$$ is sent to node $$X$$, it's buffer size is the total buffer minus the memory taken by the sum of all messages in the buffer.
|
||||
|
||||
###### Node Receptiveness
|
||||
|
||||
- Aims to avoid or decrease sending rates to the **nodes** that have higher in network delays
|
||||
|
||||
$$
|
||||
Rec(X) = \sum^N_{i=1}(T_{now} - M^i_{received}(X))
|
||||
$$
|
||||
|
||||
How long a node keeps a message before forwarding it on. If a high level of receptiveness is found on a node, it means the node isn't useful as messages aren't forwarded. Could mean the node has limited connections.
|
||||
|
||||
###### Node Congestion Rate
|
||||
|
||||
- Aims to avoid or decrease sending rates to **nodes** that congest at the higher rate
|
||||
|
||||
$$
|
||||
CR(X) = \frac{100\cdot T_{FullBuffer}(X)/T_{TotalTime}(X)}{\frac{1}{N}\cdot \sum^N_{i=1}(T_iend(X) - T_istart(X))}
|
||||
$$
|
||||
|
||||
Estimates the time between a node being full and full again. Measures the time the node is unusable.
|
||||
|
||||
#### Ego Network Congestion Metrics
|
||||
|
||||
###### Ego Network Retentiveness
|
||||
|
||||
* Aims to replicate less at the **parts of the network** with lower buffer availability.
|
||||
|
||||
$$
|
||||
EN_{Ret}(X) = \frac{1}{N}\sum^N_{i=1}Ret(C_i(X))
|
||||
$$
|
||||
|
||||
Gets the average of the retentiveness of node $$X$$ and it's neighbours $$c_i(X)$$
|
||||
|
||||
###### Ego Network Receptiveness
|
||||
|
||||
* Aims to replicate less at **parts of the network** with higher delays.
|
||||
|
||||
$$
|
||||
EN_{Rec}(X) = \frac{1}{N}\sum^N_{i=1}Rec(c_i(X))
|
||||
$$
|
||||
|
||||
###### Ego Network Congestion Rate
|
||||
|
||||
- Aims to send less to the **parts of the network** that have higher congestion rates.
|
||||
- This is useful as if a node isn't congested, but all connected nodes are. It stops it from being used.
|
||||
|
||||
$$
|
||||
EN_{CR}(X) = \frac{1}{N}\sum^N_{i=1}CR_i(X)
|
||||
$$
|
||||
|
||||
#### Contents of CAFREP Node
|
||||
|
||||

|
||||
$$
|
||||
Replication\space rate = M \times \frac{TotalUtil(Y)}{TotalUtil(X) + TotalUtil(Y)}
|
||||
$$
|
||||
Total utility, changes constantly. The replication limit grows to take advantage of all available resources, and backs off when congestion increases.
|
||||
|
||||
Social utility prevents replication at a high rate on free nodes that are not on the path to the destination.
|
||||
@@ -1,102 +0,0 @@
|
||||
# Information Centric Networks
|
||||
|
||||
#### Problems with today's Networks
|
||||
|
||||
* URLs and IP addresses are overloaded with locator and identifier functionality.
|
||||
* No consistent way to keep track of *identical copies*.
|
||||
* Information dissemination is inefficient.
|
||||
* Cannot benefit from existing copies
|
||||
* Can lead to problems like Flash-Crowd effect and Denial of service
|
||||
* Can't trust a copy received from an un-trusted node
|
||||
* Security is host-Centric
|
||||
* Based on *securing channels* (encryption) and trusting servers (authentication)
|
||||
* Application and content providers are independent of each other
|
||||
* CDNs focus on web content distributions for major players
|
||||
|
||||

|
||||
|
||||
**Important requirements for ICNs** (Information Centric Networks)
|
||||
|
||||
1. Accessing named resources - not hosts
|
||||
2. Scalable distribution through replication and caching
|
||||
3. Good control of resolution / routing and access
|
||||
|
||||
## Content-based Routing for ICNs
|
||||
|
||||
Apart from routing protocols that use direct identifiers of nodes, networking can take place based directly on content.
|
||||
|
||||
* Content can be **collected** from the network, **processed** in the network and **stored** in the network.
|
||||
* The goal is to provide a network infrastructure capable of providing services better suited to today's application requirements
|
||||
* Content distribution and mobility
|
||||
* More resilience to disruption and failures
|
||||
|
||||
#### Network Evolution
|
||||
|
||||
**Traditional networking**
|
||||
|
||||
- Host-Centric communications, addressing and end-points
|
||||
|
||||
**ICNs**
|
||||
|
||||
- Data-Centric communications addressing information
|
||||
- Decoupling in space - neither sender nor receiver need to know their partner.
|
||||
- Decoupling in time - *answer* not necessarily directly triggered by a *question*. **asynchronous communication**.
|
||||
|
||||
#### Approach
|
||||
|
||||
* Named Data Objects (NDOs)
|
||||
* In-network caching/storage
|
||||
* Multi-party communication through replication
|
||||
* Senders decoupled from receivers
|
||||
|
||||
### Dissemination Networking
|
||||
|
||||
* Data is requested by name, using any and all means available (IP, VPN tunnels, multi-cast, proxies etc)
|
||||
* Anything that hears the request and has a valid copy of the data can respond.
|
||||
* The returned data is signed, and optionally secured, so its integrity & association with name can be validated (data-Centric security)
|
||||
|
||||

|
||||
|
||||
* Change of network abstraction from **named host** to **named content** (content chunks).
|
||||
* Security is built in - **secures content** and **not the hosts**.
|
||||
* **Mobility** is present by design.
|
||||
* Can handle **static** and **dynamic** content.
|
||||
|
||||
#### Naming Data
|
||||
|
||||
###### Solution 1 - Name the data
|
||||
|
||||
- **Flat** - non human readable identifiers
|
||||
- `1HJKRH535KJH252JLH3424JLBNL`
|
||||
- **Hierarchical** - meaningful structured names
|
||||
- `/nytimes/sport/baseball/mets/game0224143`
|
||||
|
||||
###### Solution 2 - Describe the data
|
||||
|
||||
- With a set of tags
|
||||
- `baseball, new york, mets`
|
||||
- With schema that defines attributes, values and relations among attributes
|
||||
|
||||
##### Using Names in CCNs (Content Centric Networks)
|
||||
|
||||
- The hierarchical structure is used to do *longest match look-ups* which guarantees $$log(n)$$ state scaling for globally accessible data.
|
||||
- Although CCN names are longer than IP identifiers, their **explicit structure** allows look-ups as efficient as IP's.
|
||||
|
||||
### ICN Forwarding
|
||||
|
||||
* Consumer *broadcasts* and *interest* over all available communication media
|
||||
* Interest identifies a *collection of data* whose name has the interest as a prefex.
|
||||
* Anything that hears the interest and has an element of the collection can respond with that data.
|
||||
|
||||
### ICN Transport
|
||||
|
||||
* Data that matches an interest, *consumes* it.
|
||||
* Interest must be re-expressed to get new data.
|
||||
* Controlling re-expressions allows for traffic management and congestion control.
|
||||
* Multiple (distinct) interests in the same collection may be expressed
|
||||
|
||||
### ICN Caching
|
||||
|
||||
* Storage and caching are integral part of the ICN service
|
||||
* All nodes potentially have caches. Requests for data can be satisfied by any node holding a copy in it's cache.
|
||||
* ICN combines caching at the network edge with in-network caching.
|
||||
@@ -1,96 +0,0 @@
|
||||
# Content Centric Networks
|
||||
|
||||
A Brief History of Networking
|
||||
|
||||
- Gen 1. The **phone system** (focus on the **wires**)
|
||||
- The utility of the system depends on running wires to every home & office.
|
||||
- Wires are the dominant cost.
|
||||
- A *call* is not the conversation, its the **PATH** between two end-office line cards.
|
||||
- A *phone number* is not the name/address of the caller, its a **program** for the end-office switch fabric to build a path to the destination line card.
|
||||
- <img src="/lectures/acn/img/k.png" alt="switch board" style="zoom:50%;" />
|
||||
- Path building is **non-local** and **encourages centralisation** and **monopoly**.
|
||||
- Calls fail is any element in the path fails so reliability goes down exponentially as the system scales up.
|
||||
- Data cannot flow until the path is set up so efficiency decreases with setup time.
|
||||
|
||||
- Gen 2. The **Internet** (focus on the **endpoints**)
|
||||
- Data sent in independent chunks and each chunk contains the name of the final destination.
|
||||
- Nodes forward packets onward using routing tables.
|
||||
- **ARPAnet** was built on top of the existing phone system.
|
||||
|
||||
- Gen 3. **dissemination** (focus on the **data**)
|
||||
|
||||
#### TCP/IP
|
||||
|
||||
###### Pros
|
||||
|
||||
- Adaptive routing lets system **repair failures**
|
||||
- **Reliability increases exponentially** with **system size**.
|
||||
- **No call setup** means **high efficiency** at any bandwidth and scale.
|
||||
- Distributed routing supports any topology and tends to spread load and avoid a hierarchy's hot spots.
|
||||
|
||||
###### Cons
|
||||
|
||||
- *Connected* is a binary attribute.
|
||||
- Becoming part of the internet requires a globally unique, globally know IP address that's topologically stable on routing time scales.
|
||||
- Connecting is a heavy weight operation
|
||||
- The net struggles with moving nodes
|
||||
|
||||
#### Conversation and Dissemination
|
||||
|
||||
Acquiring chunks of data (web pages, emails, videos etc) is not a conversation, it's *dissemination*.
|
||||
|
||||
In a dissemination **the data matters**, not the supplier.
|
||||
|
||||
- Data is request by name.
|
||||
- Anything that hears the request, and has a valid copy can respond.
|
||||
- The return data is signed, so integrity and association can be validated.
|
||||
|
||||
CCN can run over and be run over anything e.g. IP.
|
||||
|
||||
#### CCN Packets
|
||||
|
||||

|
||||
|
||||
**Interest** - similar to HTTP `GET`
|
||||
|
||||
**Data** - similar to HTTP response
|
||||
|
||||
#### Content Based Security
|
||||
|
||||
Data packets are authenticated with digital signatures.
|
||||
|
||||

|
||||
|
||||
#### CCN Forwarding
|
||||
|
||||
Consumer *broadcasts* and *interest* over all available communication media
|
||||
|
||||
- e.g. `get '/parc.com/van/presentation.pdf'`
|
||||
- response: `heres '/parc.com/van/presentation.pdf/p1' <data>`
|
||||
|
||||
##### Names and Meaning
|
||||
|
||||
* Like IP, CCN nodes imposes no semantics on names
|
||||
* Meaning comes from **application**, **institution** and **global conventions** reflected in prefix forwarding rules.
|
||||
* Globally meaningful name leveraging the DNS global naming structure
|
||||
* `/parc.com/van/presentation.pdf`
|
||||
* Local and context sensitive, it refers to different objects depending on the room you're in.
|
||||
* `/thisRoom/projector`
|
||||
|
||||
#### Strategy Layer
|
||||
|
||||
* When you do not care who you are talking to, you don't care if they change
|
||||
* When you are not having a conversation, there's no need to migrate conversation state.
|
||||
* Multi-point gives you multi-interface for free.
|
||||
* When all communication is locally flow balanced, your stack knows exactly whats working and how well.
|
||||
|
||||
In the current Internet, Quality of Service (QoS) Problems are highly localised
|
||||
|
||||
* Roughly half the problems are from serial dependencies created by queues
|
||||
* The other half are caused from a lack of receiver based control over bottle-necked links.
|
||||
|
||||
Unlike IP, CCN is **local**, don't have queues and receivers have complete control
|
||||
|
||||

|
||||
|
||||
Tree serves as transport state
|
||||
@@ -1,138 +0,0 @@
|
||||
# Delay Tolerant Networks Security
|
||||
|
||||
#### Applications of DTNs
|
||||
|
||||
##### Interplanetary communication
|
||||
|
||||
<img src="/lectures/acn/img/o.png" alt="DTN in space" style="zoom:50%;" />
|
||||
|
||||
> **Characteristics**
|
||||
>
|
||||
> * High intermittent connectivity
|
||||
> * Extremely long message travel time
|
||||
> * Delay: finite speed of light
|
||||
> * Low Transmission reliability
|
||||
> * Inaccurate position
|
||||
> * Limited visibility
|
||||
> * Low asymmetric Data Rate
|
||||
>
|
||||
> **Security**
|
||||
>
|
||||
> - CCSDS protocol
|
||||
> - space End to End security
|
||||
> - space end to end reliability
|
||||
|
||||
##### Military
|
||||
|
||||
> No consistent network infrastructure and frequent disruptions
|
||||
>
|
||||
> **Characteristics**
|
||||
>
|
||||
> * High intermittent connectivity
|
||||
> * Mobility, destruction, noise & attacks, interference
|
||||
> * Low transmission reliability
|
||||
> * positioning inaccuracy
|
||||
> * limited visibility
|
||||
> * Low data rate
|
||||
>
|
||||
> **Security**
|
||||
>
|
||||
> - Mainly MANET security
|
||||
> - Distribution of CAs (Certificate Authorities) in mobile ad hoc networks cannot provide military level security
|
||||
> - Combining a self-organised approach with an off-line trusted third-party
|
||||
|
||||
##### Rural Areas
|
||||
|
||||
>Providing internet connectivity to rural/developing areas
|
||||
>
|
||||
>**Characteristics**
|
||||
>
|
||||
>- Intermittent connectivity
|
||||
>- Mobility - sparse development
|
||||
>- High propagation delay
|
||||
>- Asymmetric data rate
|
||||
>
|
||||
>
|
||||
>
|
||||
>**Security**
|
||||
>
|
||||
>- Standard cryptographic techniques such as PKI and transparent encrypted file systems
|
||||
|
||||
- Disaster struck areas
|
||||
- Disconnected kiosks in rural areas
|
||||
- Remote sensing applications
|
||||
|
||||
But also
|
||||
|
||||
- Bulk data distribution in urban areas
|
||||
- Sharing of individual contents in urban areas
|
||||
- Mobile location-aware sensing application
|
||||
- Social mobile applications
|
||||
|
||||
#### DTN Security Goals
|
||||
|
||||
Due to the resource-causticity that DTNs have, the focus is on protecting the DTN infrastructure from unauthorised access and use.
|
||||
|
||||
* Prevent **access** by unauthorised applications.
|
||||
* Prevent unauthorised applications from asserting control over DTN infrastructure.
|
||||
* Prevent authorised applications from sending bundles at a rate or class of service for which they **don't have permissions for**.
|
||||
* Detect and discard bundles that were sent from unauthorised applications/users.
|
||||
* Detect and discard bundles who's headers have been modified.
|
||||
* Detect and discard compromised entities.
|
||||
|
||||
Secondary emphasis is on providing optional end-to-end security services to bundle applications.
|
||||
|
||||
#### DTN Security Challenges
|
||||
|
||||
* High round-trip times and disconnections
|
||||
* Do not allow frequent distribution of a large number of certificates and encryption keys end-to-end.
|
||||
* More scalable to use user's keys and credentials at neighbouring or nearby nodes.
|
||||
* Delays or loss of connectivity to a key or certificate server
|
||||
* Multiple certificate authorities desirable but not sufficient and certificate revocation not appropriate
|
||||
* Long delays
|
||||
* Messages may be valid for days/weeks, so message expiration may not be able to be depended on to rid the network of unwanted messages as efficiently as in other types of networks.
|
||||
* Constrained Bandwidth
|
||||
* Need to minimise the cost of security in terms of network overhead (header bits).
|
||||
|
||||
###### Traditional PKI not applicable
|
||||
|
||||
* Traditional symmetric cryptography approaches are not suitable for DTNs for two major reasons
|
||||
* In PKI a user authenticates another users public key using a certificate
|
||||
* This is not possible without online access to the receivers public key or certificates
|
||||
* PKIs implement key revocation based on frequently updated online certificate revocation lists
|
||||
* In the absence of instant online access to CAs servers, a receiver cannot authenticate the sender's certificate.
|
||||
|
||||
###### Identity Based Cryptography not applicable
|
||||
|
||||
Identity Based Cryptography (IBC) schemes where the public key of each entity is replaced by its identity and associated public formatting policies are not suitable for the security in DTNs
|
||||
|
||||
- IBC does not solve the key management problem in DTNs
|
||||
- It is not scalable because it assumes that a user must know the public parameters for all the trusted parties.
|
||||
|
||||
###### Mobile ad hoc Key Management Proposals not applicable
|
||||
|
||||
- Virtual Certificate Authority
|
||||
- Not applicable due to no trusted third parties
|
||||
- Certificate chaining based on pretty good privacy (PGP)
|
||||
- Not applicable due to insufficient density of certificate graphs
|
||||
- Peer-to-peer key management based on mobilty
|
||||
- Not applicable due to certificate revocation mechanism
|
||||
|
||||
#### Existing Mandatory DTN Security
|
||||
|
||||
Based on the *bundle* protocol
|
||||
|
||||
* Hop-by-hop bundle integrity
|
||||
* Hop-by-hop bundle sender authentication
|
||||
* Access Control (only legit users with right permissions)
|
||||
* Limited protection from DoS attacks
|
||||
|
||||

|
||||
|
||||
- Payload Security Header is computed once at the source bundle agent, carried unchanged, and checked at the destination bundle agent (and possibly also security boundary bundle agents)
|
||||
|
||||
- Bundle Authentication Header is computed at every sending bundle agent and checked at every receiving hop along the way from the source to the destination.
|
||||
|
||||
Current DTN security initiative is based on pre-shared secrets and involves no trust dynamics mechanisms
|
||||
|
||||
- Works well against external threats but not applicable to internal threats
|
||||
@@ -1,40 +0,0 @@
|
||||
# Enabling Real-Time communications and Services in Heterogeneous Networks of Drones and Vehicles
|
||||
|
||||
### Real World Experiments
|
||||
|
||||
#### Agricultural Monitoring Application
|
||||
|
||||
- Agricultural context in UK
|
||||
- The production of potatoes or livestock has always been a major part of farming
|
||||
- There has always been a need for farmers to be able to observe their field crops or animals as often as possible so that they are informed quickly about potential deep rooted problems in the fields.
|
||||
|
||||
Enable mobile reliable multi-hop DTN communications (in field near Nottingham)
|
||||
|
||||
- 2 Flying drones
|
||||
- 1 Vehicle
|
||||
- 2 Static ground sensing nodes
|
||||
- Raspberry Pis
|
||||
- These capture and send data to the drones, which forward it to a node with higher computational output
|
||||
|
||||
The two static sensing nodes are deployed on two different sides of the field out of reach of each other while the drone acts as a intermediaries.
|
||||
|
||||
All sensing nodes could measure
|
||||
|
||||
- Air temp
|
||||
- wind speed
|
||||
- soil temp
|
||||
|
||||
We measure average edge to edge (E2E) delays of content query and dissemination in the network
|
||||
|
||||
- Two drones and one vehicle (3 intermediaries) result in lower delays
|
||||
|
||||
#### Smart City Applications
|
||||
|
||||
- More focused on single hop
|
||||
- 1 Hovering drone (publisher)
|
||||
- Moving vehicle (subscriber)
|
||||
- The drone continuously sends information such as sensors readings, street images, traffic videos to the vehicle which monitors road conditions
|
||||
- Single hop communications is significantly affected by physical obstructions
|
||||
- Therefore the latency went down in suburbs compared to city centres
|
||||
- Height of the drone is important as well
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
# Connecting
|
||||
|
||||
#### Elasticity: Supply and Demand
|
||||
|
||||
This is about **resource management**
|
||||
|
||||
- **Supply** - Available link capacity on path
|
||||
- **Demand** - Host transmitting and receiving traffic
|
||||
- **Elastic** - capacity reduces -> demand is scaled back
|
||||
- Hosts stop sending / send less
|
||||
- **Inelastic** - applications can’t handle this
|
||||
|
||||
TCP manages resource usage based on observed loss and latency
|
||||
|
||||
#### Quality of Service
|
||||
|
||||
If capacity > demand, there is no need for quality of service
|
||||
|
||||
If capacity < demand, we need to keep queuing minimal
|
||||
|
||||
- As queuing directly impacts latency, jitter and loss
|
||||
- In stable networks
|
||||
- **Jitter**: The difference in delays, a measure of stability
|
||||
|
||||
#### IP Type of Service
|
||||
|
||||
- Single IP header byte
|
||||
|
||||
```
|
||||
Bits 0-2: Precedence.
|
||||
Bit 3: 0 = Normal Delay, 1 = Low Delay.
|
||||
Bits 4: 0 = Normal Throughput, 1 = High Throughput.
|
||||
Bits 5: 0 = Normal Reliability, 1 = High Reliability.
|
||||
Bit 6-7: Reserved for Future Use.
|
||||
```
|
||||
|
||||
- Precedence for *special* traffic
|
||||
|
||||
```
|
||||
0 1 2 3 4 5 6 7
|
||||
+-----+-----+-----+-----+-----+-----+-----+-----+
|
||||
| | | | | | |
|
||||
| PRECEDENCE | D | T | R | 0 | 0 |
|
||||
| | | | | | |
|
||||
+-----+-----+-----+-----+-----+-----+-----+-----+
|
||||
|
||||
Precedence
|
||||
|
||||
111 - Network Control
|
||||
110 - Internetwork Control
|
||||
101 - CRITIC/ECP
|
||||
100 - Flash Override
|
||||
011 – Flash
|
||||
010 – Immediate
|
||||
001 – Priority
|
||||
000 - Routine
|
||||
```
|
||||
|
||||
### Differentiated Services (DiffServ)
|
||||
|
||||
- Operates on *traffic aggregates*
|
||||
- Label packets with desired class via ToS
|
||||
- Routers apply different queuing as operator sees fit
|
||||
- Four service classes, or *per-hop behaviour*
|
||||
- **Default**: best effort
|
||||
- No QoL applied
|
||||
- **Expedited Forwarding**: low delay, loss & jitter
|
||||
- **Assured Forwarding**: low loss if within rate
|
||||
- **Class Selector**: use ToS precedence bits
|
||||
|
||||
##### Problems
|
||||
|
||||
- End to end semantics
|
||||
- Mapping to service level agreement
|
||||
- If an internet company sells a network with a certain speed, this might have legal repercussions if QoS are enacted
|
||||
- Mapping to application demands
|
||||
|
||||
### Integrated Services (IntServ)
|
||||
|
||||
- Operates on explicitly signalled *flows*
|
||||
- Think phone switchboards
|
||||
- The network signals exactly what it can and can’t do to the destination nodes
|
||||
- Flow setup specifies some quality of service
|
||||
- Routers perform **C**onnection **A**dmission **C**ontrol
|
||||
- CDA can accept and reject traffic based on whether or not the route/path is available
|
||||
|
||||
##### Problems
|
||||
|
||||
- Complexity
|
||||
- Hard to scale
|
||||
- Mapping requirements to parameters
|
||||
- This was easier when ATM did it as they owned all the infrastructure
|
||||
- Whereas now it is difficult to map across all different companies
|
||||
- Per-flow state
|
||||
- Extremely difficult
|
||||
|
||||
## NAT
|
||||
|
||||
### Address Shortages
|
||||
|
||||
**IPv4** supports 32 bit addresses
|
||||
|
||||
- 95% allocated already (440,000 netblocks)
|
||||
|
||||
**IPv6** supports 128 bit address
|
||||
|
||||
- Loads of addresses :white_check_mark:
|
||||
- Routing protocols need to ported :negative_squared_cross_mark:
|
||||
- Associated services needing to move :negative_squared_cross_mark:
|
||||
|
||||
### Network Address Translation
|
||||
|
||||
Because IPv6 did not magically solve address shortage problem and not all routers are ipv6 aware, we had to rely on NAT.
|
||||
|
||||
- Private Addressing, `RFC1918`
|
||||
- `172.16/12`, `192.168/16`, `10/8`
|
||||
- Devices with these local IPs should never be externally routed
|
||||
- Not for security reasons - just for getting more addresses
|
||||
- Traditional NAT, `RFC3022` is the standard
|
||||
- Use private addresses internally (within the local network)
|
||||
- Map into a (small) set of routable addresses
|
||||
- Use source ports to distinguish connections
|
||||
- For large scale **carrier grade NAT** [`RFC6598`] on `100.64/10`
|
||||
|
||||
#### Implementation
|
||||
|
||||
- Requires IP, TCP/UDP header rewriting
|
||||
- Addresses, ports and checksums all need to be recalculated
|
||||
- Behaviours
|
||||
- Network Address Translation
|
||||
- Network Address and Port Translation
|
||||
|
||||
###### Full Cone
|
||||
|
||||

|
||||
```
|
||||
ea:ep - NAT address : NAT port
|
||||
```
|
||||
|
||||
When client receives packet from server 1 `da:dp`, the NAT translates the NAT address `ea:ep` to the clients internet address and port `ia:ip`.
|
||||
|
||||
###### Address Restricted Cone NAT
|
||||
|
||||

|
||||
In this case server 2 is not trusted and therefore any request will be dropped.
|
||||
|
||||
###### Port Restricted Cone NAT
|
||||
|
||||

|
||||
|
||||
If the router receives a packet from a bad IP or bad port, it will be dropped.
|
||||
|
||||
###### Symmetric NAT
|
||||
|
||||

|
||||
|
||||
Here the internal address is obfuscated from the external servers, same client can use different ports for different communications.
|
||||
@@ -1,139 +0,0 @@
|
||||
# Naming
|
||||
|
||||
IPs are not human readable.
|
||||
|
||||
Not always the appropriate granularity
|
||||
|
||||
- The address names an interface
|
||||
- This however does not give information about the kind of service / hardware
|
||||
|
||||
A file maps names to addresses
|
||||
|
||||
- Unix & Linux
|
||||
- `/etc/hosts`
|
||||
- Windows
|
||||
- `C:\Windows\System32\drivers\etc\hosts`
|
||||
|
||||
These are simple but neither automatic or scalable which led to **DNS**.
|
||||
|
||||
- Was initially `RFC882`
|
||||
- Now is `RFC1035, 1987`
|
||||
|
||||
DNS is a consistent namespace
|
||||
|
||||
- No reference to addresses, routes etc
|
||||
- Is hierarchical, distributed & cache
|
||||
- All of which to help with scalability
|
||||
- **Federated** - sources control trade-off
|
||||
- This just means DNS are worldwide
|
||||
- **Flexible** - many record
|
||||
- Simple client-server name resolution protocol
|
||||
|
||||
#### Components
|
||||
|
||||
- *Domain name space* and *resource records*
|
||||
- Tree structured name space
|
||||
- Data associated with names
|
||||
- *Name server*
|
||||
- Contains records for a sub tree
|
||||
- May cache information about any part of the tree
|
||||
- Resolver
|
||||
- Extract information from tree upon client requests
|
||||
- `gethostbyname()`
|
||||
|
||||

|
||||
|
||||
###### Root
|
||||
|
||||
- Ultimate authority with the US Dept. of commerce (NITA)
|
||||
- Managed by IANA, operated by ICANN, maintained by Verisign
|
||||
- Started with only thirteen root server clusters
|
||||
- Now much more
|
||||
- Top level Domains, TLDs
|
||||
- Operated by registrars, delegated by ICANN
|
||||
- Delegate zones to other registrars
|
||||
- and so on down the hierarchy
|
||||
- Eventually customer rents a name - their **zone**
|
||||
- Registrar installs appropriate *resource records*
|
||||
- Associated with names within the zone
|
||||
|
||||
#### Query
|
||||
|
||||
- Query generated by resolver
|
||||
- e.g. call to `gethostbyname()`, `gethostbyaddr()`
|
||||
- Carried in single UDP/53 packet
|
||||
- Or more rarely TCP/53 in case of truncation
|
||||
- UDP is not smart and therefore does not follow traffic routing (it is selfish)
|
||||
- It is beneficial for the internet as a whole to use UDP sometimes
|
||||
- Header followed by question
|
||||
- ID, Q/R, opcode, AA/TC/RD/RA, response code, counts
|
||||
- Query type, query class, query name
|
||||
|
||||
Response consists of three RRsets following the header and question
|
||||
|
||||
- **Answers**: RRs that the server had for the QNAME
|
||||
- **Authoritatives**: RRs pointing to an authority for the name
|
||||
- **Additionals**: RRs related to the question but don’t answer it
|
||||
|
||||
###### Common Resource Records
|
||||
|
||||
- `A` / `CNAME` / `PTR`
|
||||
|
||||
```
|
||||
www.cs.nott.ac.uk. 61272 IN CNAME pat.cs.nott.ac.uk.
|
||||
|
||||
pat.cs.nott.ac.uk. 68622 IN A 128.243.20.9
|
||||
|
||||
pat.cs.nott.ac.uk. 68622 IN A 128.243.21.19
|
||||
|
||||
9.20.243.128.in-addr.arpa. 39617 IN PTR pat.cs.nott.ac.uk.
|
||||
```
|
||||
|
||||
`cname` refers to the mapping of the domain name to its IP (or another domain) & ports
|
||||
|
||||
Can have 2 authoritative records
|
||||
|
||||
- `NS`
|
||||
|
||||
```
|
||||
cs.nott.ac.uk. 10585 IN NS ns1.nottingham.ac.uk.
|
||||
cs.nott.ac.uk. 10585 IN NS ns2.nottingham.ac.uk.
|
||||
cs.nott.ac.uk. 10585 IN NS marian.cs.nott.ac.uk.
|
||||
cs.nott.ac.uk. 10585 IN NS extdns1.warwick.ac.uk.
|
||||
cs.nott.ac.uk. 10585 IN NS extdns2.warwick.ac.uk.
|
||||
```
|
||||
|
||||
It is good practice to have an external DNS, UoN uses Warwick as an external DNS.
|
||||
|
||||
- `MX`
|
||||
|
||||
```
|
||||
nott.ac.uk. 3600 IN MX 1 mx191.emailfiltering.com.
|
||||
nott.ac.uk. 3600 IN MX 2 mx192.emailfiltering.com.
|
||||
nott.ac.uk 3600 IN MX 3 mx193.emailfiltering.com.
|
||||
```
|
||||
|
||||
What happens when the resolver queries a server that doesn't know the answer? two solutions:
|
||||
|
||||
1. **Iterative** (required)
|
||||
- Server responds indicating who to ask next
|
||||
- This method is slower and more difficult to retrieve an answer
|
||||
1. **Recursive** (optional)
|
||||
- Server generates a new query to the next server
|
||||
|
||||

|
||||
|
||||
#### Load Balancing
|
||||
|
||||
DNS may have multiple servers, when a query comes various algorithms can be used to choose the best one, this can be geographical location.
|
||||
|
||||
#### Operational & Security Issues
|
||||
|
||||
- Usually need primary and secondary servers
|
||||
- Separate IP netblocks, physical networks - more robust
|
||||
- DNS is a *very* common single point of failure
|
||||
- Cache poisoning
|
||||
- Caching and soft-state means bad data propagates and can persist for some time
|
||||
- Even if through simple mistakes (or of course malicious attacks)
|
||||
- Man-in-the-middle attacks
|
||||
- Can happen with both iterative & recursive queries
|
||||
@@ -1,235 +0,0 @@
|
||||
# Reliability
|
||||
|
||||
Achieving reliability:
|
||||
|
||||
- Re-transmitting lost data
|
||||
- This is done by detecting lost via explicit acknowledgment
|
||||
- These can be positive or negative
|
||||
|
||||
### Stop ‘n’ Wait
|
||||
|
||||
Simplest possible paradigm
|
||||
|
||||
- Transmit `seq(x)`
|
||||
- Wait for `ack(x)`
|
||||
- Transmit `seq(x+1)`
|
||||
|
||||

|
||||
|
||||
This has really poor performance in high latency and uses high bandwidth (half the bandwidth is overhead (acknowledgements))
|
||||
|
||||
**Rate control**: Never sending too fast for the network
|
||||
|
||||
**Sliding window**: allow unacknowledged data in flight (data to be sent)
|
||||
|
||||
**Retransmission TimeOut**: how long to wait to decide a segment is lost
|
||||
|
||||
- This requires estimates of dynamic quantities
|
||||
|
||||
- Permit N segments in flight
|
||||
- Timeout implies loss
|
||||
- Retransmit from lost packet onward
|
||||
- This is bad as imagine if only packet 3 is lost out of 5, this means client will resend 3-5.
|
||||
|
||||
##### Congestion Collapse
|
||||
|
||||
When network load is too high, it causes *congestion collapse*
|
||||
|
||||
Why?
|
||||
|
||||
- The routers buffers fill up, traffic is discarded, hosts retransmit
|
||||
- Retransmit rates increase since more data was lost
|
||||
- This was solved in “Congestion Avoidance and Control”
|
||||
|
||||
#### Stability of the Internet
|
||||
|
||||
Flows and protocols **include some sort of congestion control** and adaptation so that they moderate their bandwidth use, limit packet loss as well as get approximately fair share of available network bandwidth
|
||||
|
||||
1. **Responsiveness** defined as a number of round-trip times of sustained congestion required to reduce the rate by half
|
||||
2. **Stability and smoothness** defined as the largest reduction of the sending rate in one round trip time in a steady state scenario
|
||||
3. **Fairness** towards other flows when competing for bandwidth
|
||||
|
||||
Mimicking TCP behaviour for multimedia congestion control results in fairness towards TCP but also in significant oscillations in bandwidth
|
||||
|
||||
- Multimedia streaming applications need to **have much lower variation at throughput** over time compared to TCP to result in relatively smooth sending rates that are of importance to the end-user perceived quality.
|
||||
- The penalty for having smoother throughput than TCP while competing for bandwidth is that multimedia congestion control responds slower than TCP to changes in available bandwidth.
|
||||
- Thus, if multimedia traffic wants smooth throughput, it needs to avoid TCP’s halving of the sending rate in response to a single packet drop.
|
||||
|
||||
###### Packet Loss
|
||||
|
||||
- When choosing the method for packet loss detection, it is important to choose a method that **detects packet losses as early and accurately as possible**
|
||||
- Incorrect detection & late packet delivery can lead to incorrect packet loss estimation
|
||||
- This causes unresponsive & unfair behaviour
|
||||
- Calculating packet loss rates can be done over various lengths of time intervals.
|
||||
- Shorter intervals result in more responsive behaviour but are more susceptible to noise
|
||||
- Longer intervals = smoother but less responsive
|
||||
- It is important to find a balance
|
||||
- In order to guarantee sufficient responsiveness to congestion and preserver smoothness, methods for detecting & calculating packet loss must be chosen carefully.
|
||||
1. What mechanism can be used for packet loss detection?
|
||||
2. What algorithm can be used for packet loss rate calculation?
|
||||
3. Where can packet loss detection and calculation happen?
|
||||
|
||||
###### Approach
|
||||
|
||||
- All sent packets are marked with consecutive sequence of numbers
|
||||
- When a packet is sent a timeout value for this packet is computed and an entry containing the sequence number and the timeout value is inserted into a list and kept there until packet delivery is acknowledged or considered to be lost
|
||||
- If the timeout expires before the packet is acknowledged, the corresponding packet is considered to be lost
|
||||
- In order to adapt to varying and unpredictable network conditions, the timeout is not fixed, but computed based on one of the algorithms for TCP timeout computation
|
||||
|
||||
##### Timeout Based Approaches
|
||||
|
||||
This is mostly used for multimedia situations
|
||||
|
||||
RTT - round trip times
|
||||
|
||||
- Before the first packet is ACK and RTT measurement is made, the sender sets the TIMEOUT to a certain initial value
|
||||
- This value is usually **2.5-3 seconds for TCP**
|
||||
- For real time interactive multimedia traffic, the timeout value should be set to **0.5 seconds** as this is the time where audio delay affects media
|
||||
- When the first `RTT` measurement is taken the sender sets the smoothed `RTT` (`SRTT`), `RTT` variance (`RTTVAR`) and `TIMEOUT` in the following way
|
||||
- `SRTT = RTT`
|
||||
- `RTTVAR = RTT/2`
|
||||
- `TIMEOUT = `$\Mu\cdot$`SRTT + 4*RTTVAR`
|
||||
- Where $\Mu$ is a constant, which in this implementation is 1.08 (obtained experimentally)
|
||||
- When subsequent `RTT` measurements are made the sender sets the `RTTVAR`, `SRTT`, TIMEOUT
|
||||
- `RTTVAR`$= (1 - \frac{1}{4}) \times$`RTTVAR`$+ \frac14 \times |$`SRTT`$-$`RTT`$|$
|
||||
- `SRTT`$= (-\frac18)\times$`SRTT`$+\frac18\times$`RTT`
|
||||
- `TIMEOUT`$= \Mu\times$`SRTT`$+ 4\times$`RTTVAR`
|
||||
|
||||
###### Packet loss rate calculation
|
||||
|
||||
- Real time interactive multimedia approaches typically use the **weighted Loss Interval Average (WLIA)**
|
||||
- It relies on **using loss events** and **loss intervals** for correct computation of packet loss rate and is in accordance with how TCP performs packet loss calculation
|
||||
- A **loss event** is defined as a number of packets lost within a single RTT
|
||||
|
||||
This can be done either on the sending or receiving side
|
||||
|
||||
**Sender-side**: if the packet loss detection is done in the sender, the sender can use timeout mechanism for each packet or gap in sequence numbers of the acknowledged
|
||||
|
||||
- The receiver has to acknowledge either every packet or every packet not received
|
||||
- Acknowledging every packet can introduce high levels of traffic between between sender and receiver
|
||||
- This is solved by having receivers send report summaries of losses every nth packet or nth RTT
|
||||
|
||||
**Receiver-side**: Packet loss is detected in the receiver and explicitly reported back to the sender
|
||||
|
||||
- Noticing the gap in the sequence number - a loss event can be assumed
|
||||
- A loss event is directly forwarded to the sender
|
||||
|
||||
##### Sender vs Receiver Detection
|
||||
|
||||
Receiver driven packet loss discovery is preferred.
|
||||
|
||||
- This is because loss events are sent early as possible
|
||||
- This means high responsiveness
|
||||
|
||||
In the case of very high congestion - where there is no feedback from the receiver
|
||||
|
||||
- The pure receiver based loss detection is useless because the sender has no way of calculating packet loss
|
||||
- In these cases sender enters **self-limitation** - where packet loss is assumed and sending rate is decreased or even stopped
|
||||
|
||||
#### Adaption
|
||||
|
||||
Once the parameters of a given link are measured (packet loss and round trip times), there is a range of approaches that could be followed when choosing rate adaptation scheme(s).
|
||||
|
||||
**Equation-based control** uses a control equation that explicitly gives the maximum acceptable sending rate as a function of the recent loss event rate (loss rates).
|
||||
|
||||
**Additive Increase Multiplicative Decrease (AIMD) control** of in response to a single congestion indication.
|
||||
|
||||
###### Decision Function
|
||||
|
||||
Options for Decision function:
|
||||
|
||||
- **On congestion** (overload/packet loss/packet loss increase), **decrease the rate immediately, or periodically****
|
||||
- On absence of congestion** (underload/no packet loss, packet loss decrease), **increase the rate immediately**
|
||||
|
||||
###### Increase/decrease function
|
||||
|
||||
Options for **increase phase**: (during underload)
|
||||
|
||||
- constant additive increase rate,
|
||||
- straight jump to the expected value or value calculated by the formula
|
||||
- multiplicative increase rate
|
||||
|
||||
The default for the Internet is **constant linear increase**.
|
||||
|
||||
One could argue that a loss estimate of zero indicates that there is no congestion and thus the sending rate should be increased with the maximum possible increase factor until a loss event occurs.
|
||||
|
||||
- However, this approach **causes instabilities** in the sending rate and is very susceptible to a noisy packet drop rates.
|
||||
|
||||
Options for **decrease phase**
|
||||
|
||||
- constant multiplicative decrease factor, TCP-like or TCP-similar like.
|
||||
- linear decrease
|
||||
- straight jump to the expected value (calculated by the formula)
|
||||
|
||||
The default for the internet is multiplicative decrease (halving)
|
||||
|
||||
- Because congestion recovery should be exponential and not linear
|
||||
|
||||
Options for **decision frequency**
|
||||
|
||||
Decision frequency specifies **how often to change the rate.** **Based on system control theory, optimal adjustment frequency depends on the feedback delay.**
|
||||
|
||||
- The feedback delay **is the time between changing the rate and detecting the network’s reaction to that change.**
|
||||
- It is suggested that equation-based schemes adjust their rates **not more than once per RTT**.
|
||||
- Changing the rate too often results in oscillation
|
||||
- Infrequent change of the rate leads to an unresponsive behaviour.
|
||||
|
||||
###### Self Clocking
|
||||
|
||||
Aim is that transmission spacing matches bottleneck rate
|
||||
|
||||
- Avoids consistent queuing at bottleneck
|
||||
- Queue to smooth out short-term variation
|
||||
|
||||
##### Congestion Control
|
||||
|
||||
Aim to obey **conversation of packets**
|
||||
|
||||
- In equilibrium flow is conservative
|
||||
- New packet doesn't enter until one leaves
|
||||
|
||||
This fails in three ways:
|
||||
|
||||
1. Connection doesn't reach equilibrium
|
||||
2. Sender transmits too soon
|
||||
3. Resource limits prevent equilibrium being reached
|
||||
|
||||
Solutions:
|
||||
|
||||
**Slow-start**
|
||||
|
||||
- Each ACK opens congestion window by 1 packet
|
||||
- Every ACK, `cwnd += 1`
|
||||
- Every RTT `cwnd *= 2`
|
||||
- If a stop occurs, stop or `cwnd == ssthresh`
|
||||
- Else multiplicative increase
|
||||
|
||||

|
||||
|
||||
**Congestion Avoidance**
|
||||
|
||||
1. Network signals congestion occurring
|
||||
- Detect loss
|
||||
2. Host responds by reducing sending rate
|
||||
- `ssthresh := cwnd/2` multiplicative decrease
|
||||
- `cwnd := 1` initialises slow start
|
||||
|
||||
Avoid congestion by slow increase
|
||||
|
||||
`cwnd += 1/cwnd` window increases 1 per window
|
||||
|
||||
TCP is not always useful
|
||||
|
||||
- Reliability can cause untimely delivery
|
||||
|
||||
Audio/Video codecs usually produce frames (not continuous bytestream)
|
||||
|
||||
- Losing a frame is better than delaying all subsequent data
|
||||
|
||||
UDP encapsulates media using **R**eal **T**ime **P**rotocol
|
||||
|
||||
- Sequencing, time stamping, delivery monitoring, no quality of service
|
||||
- Adds a control channel
|
||||
- Back channel to report statistics & participants
|
||||
- Transport only
|
||||
- Leaves encodings & floor control to application
|
||||
@@ -1,163 +0,0 @@
|
||||
# Border Gateway Protocol
|
||||
|
||||
#### Routing Protocols
|
||||
|
||||
The main job is to distribute the data to build forwarding tables
|
||||
|
||||
- These are **intra-domain routing protocols**
|
||||
- Or **Interior gateway protocols**
|
||||
- When the source and destination are **inside** the **same network**
|
||||
|
||||
It is important to distinguish between local and global protocols
|
||||
|
||||
- Interior vs Exterior Gateway Protocol (IGP vs EGP)
|
||||
|
||||
##### BGPv4
|
||||
|
||||
The internet inter-domain routing protocol
|
||||
|
||||
- Derives from GGP & EGP
|
||||
- Deals in IP prefixes and **autonomous systems**
|
||||
- autonomous systems are purely administrative
|
||||
- Purpose is to enable *policy* to be applied
|
||||
- Only prefixes matter in the data-plane
|
||||
- Internet policy domains
|
||||
- Logical construct only
|
||||
- No meaning outside BGP
|
||||
- Do not map simply onto ISPs or networks
|
||||
- Currently ~493,000 prefixes & ~46,000 ASs
|
||||
- Because we have less ASs, the routing is easily -> less complex
|
||||
- Reduces complexity
|
||||
- Speeds up performance
|
||||
|
||||
BGP uses TCP as transport
|
||||
|
||||
- `OPEN`, `UPDATE`, `KEEPALIVE`, `NOTIFICATION`
|
||||
|
||||
Sessions between peers have:
|
||||
|
||||
- Simple capability negotiation
|
||||
- Manage simultaneous `OPEN` connections
|
||||
- Lose everything on session failure
|
||||
|
||||
#### Sessions and Routing Information Base (RIBs)
|
||||
|
||||
A BGP peer typically has many sessions
|
||||
|
||||
- Logically, for each peer, it receives the information about routing from peers, this is sorted into `Adj-RIB-in` table.
|
||||
- After processing, it produces a `Adj-RIB-out` table which it sends to other peers
|
||||
- Advertisements received and to be sent
|
||||
- Generates a local RIB table from `Adj-RIB-in`
|
||||
- Routes to use and potentially distribute
|
||||
- Resolved into per-port forwarding tables
|
||||
- Generate `Adj-RIB-out` from `Loc-RIB` and policy
|
||||
|
||||
#### Update messages
|
||||
|
||||
- Incremental - indicate *changes* to state
|
||||
- These updates could be:
|
||||
- Withdrawn routes
|
||||
- Path attributes, common to all advertised routes
|
||||
- Advertised routes, known as NLRI
|
||||
- There are ~27 path attributes
|
||||
- Only ~12 are in common use
|
||||
- Communicate information about prefixes
|
||||
- Used to apply policy in BGP *decision process*
|
||||
|
||||
##### Path Attributes
|
||||
|
||||
Mandatory - every AS has to inform the other ASs about these 3 attributes:
|
||||
|
||||
- Next hop
|
||||
- AS Path
|
||||
- Origin
|
||||
|
||||
Discretionary
|
||||
|
||||
- Local preferences
|
||||
- Allows prioritisation of ASs
|
||||
|
||||
Optional & transitive
|
||||
|
||||
- Aggregator
|
||||
- Community
|
||||
- Extended Communities
|
||||
|
||||
Optional & non-transitive
|
||||
|
||||
- Multi-exit discriminator
|
||||
- Originator ID
|
||||
|
||||
### Path Vectors
|
||||
|
||||
**Distance Vector** - prefer lowest cost path (not always)
|
||||
|
||||
**Path Vector**
|
||||
|
||||
- How do we know if an AS has seen this advert before
|
||||
- Store the list of ASs in the packet
|
||||
- This is called the `AS_PATH`
|
||||
- This way loops can be broken
|
||||
- If our ASN appears in a received `AS_PATH`, drop the advert
|
||||
|
||||
##### Decision Process
|
||||
|
||||
Drop prefix if:
|
||||
|
||||
- `NEXT_HOP` is unreachable via local routing table
|
||||
- Local AS appears in `AS_PATH` (packet in a loop)
|
||||
|
||||
Then (commonly) apply following preference:
|
||||
|
||||
1. Higher `weight` (local to this router)
|
||||
2. Highest `LOCAL_PREF`
|
||||
3. Shortest `AS_PATH`
|
||||
4. Lowest `ORIGIN`
|
||||
5. Lowest `MED`
|
||||
6. `EGP` to `IGP` (hot potato)
|
||||
7. Shortest internal path
|
||||
8. Prefer oldest route
|
||||
- Oldest routes are often most stable
|
||||
9. Lowest interface IP address
|
||||
|
||||
### Consistency
|
||||
|
||||
Learn external routes on `EBGP` sessions
|
||||
|
||||
- `EBGP` defined as peers having different ASNs
|
||||
- Must ensure every router knows all external routes
|
||||
- Redistribute external routes inside the network
|
||||
- Via `IGP` - only in small networks
|
||||
- via `IBGP` - gives full control over route distribution
|
||||
|
||||
###### Scaling
|
||||
|
||||
Can distribute `IBGP` routes on `IBGP` sessions
|
||||
|
||||
- Have to maintain $N\cdot \frac{(N-1)}{2}$ `IBGP` sessions
|
||||
- Each carrying up to 490k routes x2 tables
|
||||
- Two standard solutions
|
||||
1. **Route Reflectors**
|
||||
- Super nodes re-advertising `IBGP` routes
|
||||
- Allows for hierarchy
|
||||
2. **AS Confederations**
|
||||
- split AS up into mini-ASs
|
||||
|
||||
###### Failures
|
||||
|
||||
- Handling link failures
|
||||
- Bind to loopback
|
||||
- If it cant talk to other nodes, will only support communication internally
|
||||
- Flap damping
|
||||
- A warning message saying don't send traffic to me
|
||||
- This can make things worse if this message is delayed
|
||||
- Process failures
|
||||
- Out of memory error due to too many routes
|
||||
|
||||
##### Network Inter-connection
|
||||
|
||||
- Networks interconnect via `EBGP` sessions
|
||||
- POPs - points of presence or IX internet exchanges
|
||||
- Multi-homing
|
||||
- This is all logical
|
||||
- http://0x0.st/ooCs.png
|
||||
|
Before Width: | Height: | Size: 19 KiB |
|
Before Width: | Height: | Size: 120 KiB |
|
Before Width: | Height: | Size: 53 KiB |
|
Before Width: | Height: | Size: 310 KiB |
|
Before Width: | Height: | Size: 29 KiB |
|
Before Width: | Height: | Size: 134 KiB |
|
Before Width: | Height: | Size: 128 KiB |
|
Before Width: | Height: | Size: 126 KiB |
|
Before Width: | Height: | Size: 62 KiB |
|
Before Width: | Height: | Size: 26 KiB |
|
Before Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 110 KiB |
|
Before Width: | Height: | Size: 146 KiB |
|
Before Width: | Height: | Size: 62 KiB |
|
Before Width: | Height: | Size: 500 KiB |
|
Before Width: | Height: | Size: 110 KiB |
|
Before Width: | Height: | Size: 78 KiB |
|
Before Width: | Height: | Size: 49 KiB |
|
Before Width: | Height: | Size: 1.3 MiB |