stringreplacer_test.gox 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422
  1. // Copyright 2009 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package misspell_test
  5. import (
  6. "bytes"
  7. "fmt"
  8. "strings"
  9. "testing"
  10. . "github.com/client9/misspell"
  11. )
  12. var htmlEscaper = NewStringReplacer(
  13. "&", "&",
  14. "<", "&lt;",
  15. ">", "&gt;",
  16. `"`, "&quot;",
  17. "'", "&apos;",
  18. )
  19. var htmlUnescaper = NewStringReplacer(
  20. "&amp;", "&",
  21. "&lt;", "<",
  22. "&gt;", ">",
  23. "&quot;", `"`,
  24. "&apos;", "'",
  25. )
  26. // The http package's old HTML escaping function.
  27. func oldHTMLEscape(s string) string {
  28. s = strings.Replace(s, "&", "&amp;", -1)
  29. s = strings.Replace(s, "<", "&lt;", -1)
  30. s = strings.Replace(s, ">", "&gt;", -1)
  31. s = strings.Replace(s, `"`, "&quot;", -1)
  32. s = strings.Replace(s, "'", "&apos;", -1)
  33. return s
  34. }
  35. var capitalLetters = NewStringReplacer("a", "A", "b", "B")
  36. // TestReplacer tests the replacer implementations.
  37. func TestReplacer(t *testing.T) {
  38. type testCase struct {
  39. r *StringReplacer
  40. in, out string
  41. }
  42. var testCases []testCase
  43. // str converts 0xff to "\xff". This isn't just string(b) since that converts to UTF-8.
  44. str := func(b byte) string {
  45. return string([]byte{b})
  46. }
  47. var s []string
  48. // inc maps "\x00"->"\x01", ..., "a"->"b", "b"->"c", ..., "\xff"->"\x00".
  49. for i := 0; i < 256; i++ {
  50. s = append(s, str(byte(i)), str(byte(i+1)))
  51. }
  52. inc := NewStringReplacer(s...)
  53. // Test cases with 1-byte old strings, 1-byte new strings.
  54. testCases = append(testCases,
  55. testCase{capitalLetters, "brad", "BrAd"},
  56. testCase{capitalLetters, strings.Repeat("a", (32<<10)+123), strings.Repeat("A", (32<<10)+123)},
  57. testCase{capitalLetters, "", ""},
  58. testCase{inc, "brad", "csbe"},
  59. testCase{inc, "\x00\xff", "\x01\x00"},
  60. testCase{inc, "", ""},
  61. testCase{NewStringReplacer("a", "1", "a", "2"), "brad", "br1d"},
  62. )
  63. // repeat maps "a"->"a", "b"->"bb", "c"->"ccc", ...
  64. s = nil
  65. for i := 0; i < 256; i++ {
  66. n := i + 1 - 'a'
  67. if n < 1 {
  68. n = 1
  69. }
  70. s = append(s, str(byte(i)), strings.Repeat(str(byte(i)), n))
  71. }
  72. repeat := NewStringReplacer(s...)
  73. // Test cases with 1-byte old strings, variable length new strings.
  74. testCases = append(testCases,
  75. testCase{htmlEscaper, "No changes", "No changes"},
  76. testCase{htmlEscaper, "I <3 escaping & stuff", "I &lt;3 escaping &amp; stuff"},
  77. testCase{htmlEscaper, "&&&", "&amp;&amp;&amp;"},
  78. testCase{htmlEscaper, "", ""},
  79. testCase{repeat, "brad", "bbrrrrrrrrrrrrrrrrrradddd"},
  80. testCase{repeat, "abba", "abbbba"},
  81. testCase{repeat, "", ""},
  82. testCase{NewStringReplacer("a", "11", "a", "22"), "brad", "br11d"},
  83. )
  84. // The remaining test cases have variable length old strings.
  85. testCases = append(testCases,
  86. testCase{htmlUnescaper, "&amp;amp;", "&amp;"},
  87. testCase{htmlUnescaper, "&lt;b&gt;HTML&apos;s neat&lt;/b&gt;", "<b>HTML's neat</b>"},
  88. testCase{htmlUnescaper, "", ""},
  89. testCase{NewStringReplacer("a", "1", "a", "2", "xxx", "xxx"), "brad", "br1d"},
  90. testCase{NewStringReplacer("a", "1", "aa", "2", "aaa", "3"), "aaaa", "1111"},
  91. testCase{NewStringReplacer("aaa", "3", "aa", "2", "a", "1"), "aaaa", "31"},
  92. )
  93. // gen1 has multiple old strings of variable length. There is no
  94. // overall non-empty common prefix, but some pairwise common prefixes.
  95. gen1 := NewStringReplacer(
  96. "aaa", "3[aaa]",
  97. "aa", "2[aa]",
  98. "a", "1[a]",
  99. "i", "i",
  100. "longerst", "most long",
  101. "longer", "medium",
  102. "long", "short",
  103. "xx", "xx",
  104. "x", "X",
  105. "X", "Y",
  106. "Y", "Z",
  107. )
  108. testCases = append(testCases,
  109. testCase{gen1, "fooaaabar", "foo3[aaa]b1[a]r"},
  110. testCase{gen1, "long, longerst, longer", "short, most long, medium"},
  111. testCase{gen1, "xxxxx", "xxxxX"},
  112. testCase{gen1, "XiX", "YiY"},
  113. testCase{gen1, "", ""},
  114. )
  115. // gen2 has multiple old strings with no pairwise common prefix.
  116. gen2 := NewStringReplacer(
  117. "roses", "red",
  118. "violets", "blue",
  119. "sugar", "sweet",
  120. )
  121. testCases = append(testCases,
  122. testCase{gen2, "roses are red, violets are blue...", "red are red, blue are blue..."},
  123. testCase{gen2, "", ""},
  124. )
  125. // gen3 has multiple old strings with an overall common prefix.
  126. gen3 := NewStringReplacer(
  127. "abracadabra", "poof",
  128. "abracadabrakazam", "splat",
  129. "abraham", "lincoln",
  130. "abrasion", "scrape",
  131. "abraham", "isaac",
  132. )
  133. testCases = append(testCases,
  134. testCase{gen3, "abracadabrakazam abraham", "poofkazam lincoln"},
  135. testCase{gen3, "abrasion abracad", "scrape abracad"},
  136. testCase{gen3, "abba abram abrasive", "abba abram abrasive"},
  137. testCase{gen3, "", ""},
  138. )
  139. // foo{1,2,3,4} have multiple old strings with an overall common prefix
  140. // and 1- or 2- byte extensions from the common prefix.
  141. foo1 := NewStringReplacer(
  142. "foo1", "A",
  143. "foo2", "B",
  144. "foo3", "C",
  145. )
  146. foo2 := NewStringReplacer(
  147. "foo1", "A",
  148. "foo2", "B",
  149. "foo31", "C",
  150. "foo32", "D",
  151. )
  152. foo3 := NewStringReplacer(
  153. "foo11", "A",
  154. "foo12", "B",
  155. "foo31", "C",
  156. "foo32", "D",
  157. )
  158. foo4 := NewStringReplacer(
  159. "foo12", "B",
  160. "foo32", "D",
  161. )
  162. testCases = append(testCases,
  163. testCase{foo1, "fofoofoo12foo32oo", "fofooA2C2oo"},
  164. testCase{foo1, "", ""},
  165. testCase{foo2, "fofoofoo12foo32oo", "fofooA2Doo"},
  166. testCase{foo2, "", ""},
  167. testCase{foo3, "fofoofoo12foo32oo", "fofooBDoo"},
  168. testCase{foo3, "", ""},
  169. testCase{foo4, "fofoofoo12foo32oo", "fofooBDoo"},
  170. testCase{foo4, "", ""},
  171. )
  172. // genAll maps "\x00\x01\x02...\xfe\xff" to "[all]", amongst other things.
  173. allBytes := make([]byte, 256)
  174. for i := range allBytes {
  175. allBytes[i] = byte(i)
  176. }
  177. allString := string(allBytes)
  178. genAll := NewStringReplacer(
  179. allString, "[all]",
  180. "\xff", "[ff]",
  181. "\x00", "[00]",
  182. )
  183. testCases = append(testCases,
  184. testCase{genAll, allString, "[all]"},
  185. testCase{genAll, "a\xff" + allString + "\x00", "a[ff][all][00]"},
  186. testCase{genAll, "", ""},
  187. )
  188. // Test cases with empty old strings.
  189. blankToX1 := NewStringReplacer("", "X")
  190. blankToX2 := NewStringReplacer("", "X", "", "")
  191. blankHighPriority := NewStringReplacer("", "X", "o", "O")
  192. blankLowPriority := NewStringReplacer("o", "O", "", "X")
  193. blankNoOp1 := NewStringReplacer("", "")
  194. blankNoOp2 := NewStringReplacer("", "", "", "A")
  195. blankFoo := NewStringReplacer("", "X", "foobar", "R", "foobaz", "Z")
  196. testCases = append(testCases,
  197. testCase{blankToX1, "foo", "XfXoXoX"},
  198. testCase{blankToX1, "", "X"},
  199. testCase{blankToX2, "foo", "XfXoXoX"},
  200. testCase{blankToX2, "", "X"},
  201. testCase{blankHighPriority, "oo", "XOXOX"},
  202. testCase{blankHighPriority, "ii", "XiXiX"},
  203. testCase{blankHighPriority, "oiio", "XOXiXiXOX"},
  204. testCase{blankHighPriority, "iooi", "XiXOXOXiX"},
  205. testCase{blankHighPriority, "", "X"},
  206. testCase{blankLowPriority, "oo", "OOX"},
  207. testCase{blankLowPriority, "ii", "XiXiX"},
  208. testCase{blankLowPriority, "oiio", "OXiXiOX"},
  209. testCase{blankLowPriority, "iooi", "XiOOXiX"},
  210. testCase{blankLowPriority, "", "X"},
  211. testCase{blankNoOp1, "foo", "foo"},
  212. testCase{blankNoOp1, "", ""},
  213. testCase{blankNoOp2, "foo", "foo"},
  214. testCase{blankNoOp2, "", ""},
  215. testCase{blankFoo, "foobarfoobaz", "XRXZX"},
  216. testCase{blankFoo, "foobar-foobaz", "XRX-XZX"},
  217. testCase{blankFoo, "", "X"},
  218. )
  219. // single string replacer
  220. abcMatcher := NewStringReplacer("abc", "[match]")
  221. testCases = append(testCases,
  222. testCase{abcMatcher, "", ""},
  223. testCase{abcMatcher, "ab", "ab"},
  224. testCase{abcMatcher, "abc", "[match]"},
  225. testCase{abcMatcher, "abcd", "[match]d"},
  226. testCase{abcMatcher, "cabcabcdabca", "c[match][match]d[match]a"},
  227. )
  228. // Issue 6659 cases (more single string replacer)
  229. noHello := NewStringReplacer("Hello", "")
  230. testCases = append(testCases,
  231. testCase{noHello, "Hello", ""},
  232. testCase{noHello, "Hellox", "x"},
  233. testCase{noHello, "xHello", "x"},
  234. testCase{noHello, "xHellox", "xx"},
  235. )
  236. // No-arg test cases.
  237. nop := NewStringReplacer()
  238. testCases = append(testCases,
  239. testCase{nop, "abc", "abc"},
  240. testCase{nop, "", ""},
  241. )
  242. // Run the test cases.
  243. for i, tc := range testCases {
  244. if s := tc.r.Replace(tc.in); s != tc.out {
  245. t.Errorf("%d. strings.Replace(%q) = %q, want %q", i, tc.in, s, tc.out)
  246. }
  247. var buf bytes.Buffer
  248. n, err := tc.r.WriteString(&buf, tc.in)
  249. if err != nil {
  250. t.Errorf("%d. WriteString: %v", i, err)
  251. continue
  252. }
  253. got := buf.String()
  254. if got != tc.out {
  255. t.Errorf("%d. WriteString(%q) wrote %q, want %q", i, tc.in, got, tc.out)
  256. continue
  257. }
  258. if n != len(tc.out) {
  259. t.Errorf("%d. WriteString(%q) wrote correct string but reported %d bytes; want %d (%q)",
  260. i, tc.in, n, len(tc.out), tc.out)
  261. }
  262. }
  263. }
  264. type errWriter struct{}
  265. func (errWriter) Write(p []byte) (n int, err error) {
  266. return 0, fmt.Errorf("unwritable")
  267. }
  268. func BenchmarkGenericNoMatch(b *testing.B) {
  269. str := strings.Repeat("A", 100) + strings.Repeat("B", 100)
  270. generic := NewStringReplacer("a", "A", "b", "B", "12", "123") // varying lengths forces generic
  271. for i := 0; i < b.N; i++ {
  272. generic.Replace(str)
  273. }
  274. }
  275. func BenchmarkGenericMatch1(b *testing.B) {
  276. str := strings.Repeat("a", 100) + strings.Repeat("b", 100)
  277. generic := NewStringReplacer("a", "A", "b", "B", "12", "123")
  278. for i := 0; i < b.N; i++ {
  279. generic.Replace(str)
  280. }
  281. }
  282. func BenchmarkGenericMatch2(b *testing.B) {
  283. str := strings.Repeat("It&apos;s &lt;b&gt;HTML&lt;/b&gt;!", 100)
  284. for i := 0; i < b.N; i++ {
  285. htmlUnescaper.Replace(str)
  286. }
  287. }
  288. func benchmarkSingleString(b *testing.B, pattern, text string) {
  289. r := NewStringReplacer(pattern, "[match]")
  290. b.SetBytes(int64(len(text)))
  291. b.ResetTimer()
  292. for i := 0; i < b.N; i++ {
  293. r.Replace(text)
  294. }
  295. }
  296. func BenchmarkSingleMaxSkipping(b *testing.B) {
  297. benchmarkSingleString(b, strings.Repeat("b", 25), strings.Repeat("a", 10000))
  298. }
  299. func BenchmarkSingleLongSuffixFail(b *testing.B) {
  300. benchmarkSingleString(b, "b"+strings.Repeat("a", 500), strings.Repeat("a", 1002))
  301. }
  302. func BenchmarkSingleMatch(b *testing.B) {
  303. benchmarkSingleString(b, "abcdef", strings.Repeat("abcdefghijklmno", 1000))
  304. }
  305. func BenchmarkByteByteNoMatch(b *testing.B) {
  306. str := strings.Repeat("A", 100) + strings.Repeat("B", 100)
  307. for i := 0; i < b.N; i++ {
  308. capitalLetters.Replace(str)
  309. }
  310. }
  311. func BenchmarkByteByteMatch(b *testing.B) {
  312. str := strings.Repeat("a", 100) + strings.Repeat("b", 100)
  313. for i := 0; i < b.N; i++ {
  314. capitalLetters.Replace(str)
  315. }
  316. }
  317. func BenchmarkByteStringMatch(b *testing.B) {
  318. str := "<" + strings.Repeat("a", 99) + strings.Repeat("b", 99) + ">"
  319. for i := 0; i < b.N; i++ {
  320. htmlEscaper.Replace(str)
  321. }
  322. }
  323. func BenchmarkHTMLEscapeNew(b *testing.B) {
  324. str := "I <3 to escape HTML & other text too."
  325. for i := 0; i < b.N; i++ {
  326. htmlEscaper.Replace(str)
  327. }
  328. }
  329. func BenchmarkHTMLEscapeOld(b *testing.B) {
  330. str := "I <3 to escape HTML & other text too."
  331. for i := 0; i < b.N; i++ {
  332. oldHTMLEscape(str)
  333. }
  334. }
  335. func BenchmarkByteStringReplacerWriteString(b *testing.B) {
  336. str := strings.Repeat("I <3 to escape HTML & other text too.", 100)
  337. buf := new(bytes.Buffer)
  338. for i := 0; i < b.N; i++ {
  339. htmlEscaper.WriteString(buf, str)
  340. buf.Reset()
  341. }
  342. }
  343. func BenchmarkByteReplacerWriteString(b *testing.B) {
  344. str := strings.Repeat("abcdefghijklmnopqrstuvwxyz", 100)
  345. buf := new(bytes.Buffer)
  346. for i := 0; i < b.N; i++ {
  347. capitalLetters.WriteString(buf, str)
  348. buf.Reset()
  349. }
  350. }
  351. // BenchmarkByteByteReplaces compares byteByteImpl against multiple Replaces.
  352. func BenchmarkByteByteReplaces(b *testing.B) {
  353. str := strings.Repeat("a", 100) + strings.Repeat("b", 100)
  354. for i := 0; i < b.N; i++ {
  355. strings.Replace(strings.Replace(str, "a", "A", -1), "b", "B", -1)
  356. }
  357. }