11package main
22
33import (
4+ "bytes"
45 "encoding/json"
56 "html"
7+ "io"
68 "log"
79 "net/http"
810 "net/url"
@@ -267,11 +269,14 @@ func TibiadataHTMLDataCollectorV3(TibiaURL string) string {
267269 }
268270 }
269271
270- // Convert string to io.Reader
271- res_io := strings .NewReader (res .String ())
272+ // Convert body to io.Reader
273+ resIo := bytes .NewReader (res .Body ())
274+
275+ // wrap reader in a converting reader from ISO 8859-1 to UTF-8
276+ resIo2 := TibiaDataConvertEncodingtoUTF8 (resIo )
272277
273278 // Load the HTML document
274- doc , err := goquery .NewDocumentFromReader (res_io )
279+ doc , err := goquery .NewDocumentFromReader (resIo2 )
275280 if err != nil {
276281 log .Printf ("[error] TibiadataHTMLDataCollectorV3 (URL: %s) error: %s" , TibiaURL , err )
277282 }
@@ -282,30 +287,20 @@ func TibiadataHTMLDataCollectorV3(TibiaURL string) string {
282287 log .Fatal (err )
283288 }
284289
285- // convert string from eg " " to " "
286- data = html .UnescapeString (data )
287- data = strings .ReplaceAll (data , " " , " " )
288-
289- // convert string from ISO 8859-1 to UTF-8
290- data , _ = TibiaDataConvertEncodingtoUTF8 (data )
291-
292290 // Return of extracted html to functions..
293- return string ( data )
291+ return data
294292}
295293
296294// TibiadataHTMLRemoveLinebreaksV3 func
297295func TibiadataHTMLRemoveLinebreaksV3 (data string ) string {
298- return string ( strings .ReplaceAll (data , "\n " , "" ) )
296+ return strings .ReplaceAll (data , "\n " , "" )
299297}
300298
301299// TibiadataRemoveURLsV3 func
302300func TibiadataRemoveURLsV3 (data string ) string {
303301 // prepare return value
304302 var returnData string
305303
306- // convert string from UTF8 to ISO88591
307- data , _ = TibiaDataConvertEncodingtoISO88591 (data )
308-
309304 // Regex to remove URLs
310305 regex := regexp .MustCompile (`<a.*>(.*)<\/a>` )
311306 result := regex .FindAllStringSubmatch (data , - 1 )
@@ -315,34 +310,29 @@ func TibiadataRemoveURLsV3(data string) string {
315310 } else {
316311 returnData = ""
317312 }
318- return string ( returnData )
313+ return returnData
319314}
320315
321316// TibiadataStringWorldFormatToTitleV3 func
322317func TibiadataStringWorldFormatToTitleV3 (world string ) string {
323- return string ( strings .Title (strings .ToLower (world ) ))
318+ return strings .Title (strings .ToLower (world ))
324319}
325320
326321// TibiadataUnescapeStringV3 func
327322func TibiadataUnescapeStringV3 (data string ) string {
328- // data, _ = TibiaDataConvertEncodingtoUTF8(data)
329- return string (html .UnescapeString (data ))
323+ return html .UnescapeString (data )
330324}
331325
332326// TibiadataQueryEscapeStringV3 func
333327func TibiadataQueryEscapeStringV3 (data string ) string {
334328 data , _ = TibiaDataConvertEncodingtoISO88591 (data )
335- return string ( url .QueryEscape (data ) )
329+ return url .QueryEscape (data )
336330}
337331
338332// TibiadataDatetimeV3 func
339333func TibiadataDatetimeV3 (date string ) string {
340-
341334 var returnDate string
342335
343- // we need to use TibiaDataConvertEncodingtoISO88591 so that the parser doens't complain
344- date , _ = TibiaDataConvertEncodingtoISO88591 (date )
345-
346336 // If statement to determine if date string is filled or empty
347337 if date == "" {
348338 // The string that should be returned is the current timestamp
@@ -389,9 +379,6 @@ func TibiadataDatetimeV3(date string) string {
389379
390380// TibiadataDateV3 func
391381func TibiadataDateV3 (date string ) string {
392- // we need to use TibiaDataConvertEncodingtoISO88591 so that the parser doens't complain
393- date , _ = TibiaDataConvertEncodingtoISO88591 (date )
394-
395382 // use regex to skip weird formatting on "spaces"
396383 regex1 := regexp .MustCompile (`([a-zA-Z]{3}).*([0-9]{2}).*([0-9]{4})` )
397384 subma1 := regex1 .FindAllStringSubmatch (date , - 1 )
@@ -416,7 +403,7 @@ func TibiadataStringToIntegerV3(data string) int {
416403 returnData , _ := strconv .Atoi (processedString )
417404
418405 // Return of formatted date and time string to functions..
419- return int ( returnData )
406+ return returnData
420407}
421408
422409// match html tag and replace it with ""
@@ -444,9 +431,8 @@ func TibiaDataConvertEncodingtoISO88591(data string) (string, error) {
444431}
445432
446433// TibiaDataConvertEncodingtoUTF8 func - convert string from latin1 (ISO 8859-1) to UTF-8
447- func TibiaDataConvertEncodingtoUTF8 (data string ) (string , error ) {
448- data , err := charmap .ISO8859_1 .NewDecoder ().String (data )
449- return data , err
434+ func TibiaDataConvertEncodingtoUTF8 (data io.Reader ) io.Reader {
435+ return charmap .ISO8859_1 .NewDecoder ().Reader (data )
450436}
451437
452438// isEnvExist func - check if environment var is set
@@ -457,8 +443,13 @@ func isEnvExist(key string) bool {
457443 return false
458444}
459445
460- func TibiaDataSanitizeString (data string ) string {
461- data = html .UnescapeString (data )
446+ // TibiaDataSanitizeEscapedString func - run unescape string on string
447+ func TibiaDataSanitizeEscapedString (data string ) string {
448+ return html .UnescapeString (data )
449+ }
450+
451+ // TibiaDataSanitizeDoubleQuoteString func - replaces double quotes to single quotes in strings
452+ func TibiaDataSanitizeDoubleQuoteString (data string ) string {
462453 return strings .ReplaceAll (data , "\" " , "'" )
463454}
464455
0 commit comments