Charsets in web scripts

[lordzoster]

Hallo
I wrote a javascript web script that gets a JSON from an external web service: the JSON is encoded by PHP json_encode and is printed with a charset of UTF-8, Content-type: application/json.
Some of the values contain accented characters (Italian but even other european languages).

I'm using the AlfrescoE4Xutils.jar for performing the XHR read, XMLHttpRequest method.

If I print out in the result in an .html Freemarker Template (charset: utf-8) I get weird accented characters. Eg. è becomes ì or the like, meaning that somewhere along the path there has been somewhat an UTF-8 to Latin1 to ISO 8859-1 to UTF-8 conversion.

Somewhere I read of people having similar problems in Spring-based application, and they have been required to change Spring configuration properties.

Due to the huge number of layers involved, I admit I have no idea where to look for.
Linux env? Tomcat? Spring? Rhino? FreeMarker? AlfrescoE4Xutils.jar library?

[lordzoster]

to avoid depending from platforms and charsets and languages and other amenities, I output a unicode-encoded JSON (in the 4-chars form, \uxxxx, eg. \u00e0 means "è"); then using Rishida's conversion functions I decode them back, using String.replace(regex) and an hex2char.

Then i eval the JSON string.

Here the web script components - I left every comment in the controller in case it could help:
[The controller
<javascript>
var simplehttpresult = "no result";
var xmlhttpresult = "no result";
var error = "no error";
var mystring = "My String";

function hex2char ( hex ) {
   // converts a single hex number to a character
   // note that no checking is performed to ensure that this is just a hex number, eg. no spaces etc
   // hex: string, the hex codepoint to be converted
   var result = '';
   var n = parseInt(hex, 16);
    if (n <= 0xFFFF) { result += String.fromCharCode(n); }
   else if (n <= 0x10FFFF) {
      n -= 0x10000
      result += String.fromCharCode(0xD800 | (n >> 10)) + String.fromCharCode(0xDC00 | (n & 0x3FF));
       }
   else { result += 'hex2Char error: Code point out of range: '+dec2hex(n); }
   return result;
}

function convertjEsc2Char ( str, shortEscapes ) {
   // converts a string containing JavaScript or Java escapes to a string of characters
   // str: string, the input
   // shortEscapes: boolean, if true the function will convert \b etc to characters
   
   // convert \U and 6 digit escapes to characters
   /*str = str.replace(/\\U([A-Fa-f0-9]{8})/g,
               function(matchstr, parens) {
                  return hex2char(parens);
                  }
                  );
   */
   // mandatory since Rhino doesn't know how to handle the string,
   // it thinks it is java and replace method doesn't accept regex
   var str = String(str);
   // convert \u and 6 digit escapes to characters
   try {
      str = str.replace(/\\u([A-Fa-f0-9]{4})/g,
                  function(matchstr, parens) {
                     return hex2char(parens);
                     }
                     );
      // convert \b etc to characters, if flag set
      if (shortEscapes) {
         //str = str.replace(/\\0/g, '\0');
         str = str.replace(/\\b/g, '\b');
         str = str.replace(/\\t/g, '\t');
         str = str.replace(/\\n/g, '\n');
         str = str.replace(/\\v/g, '\v');
         str = str.replace(/\\f/g, '\f');
         str = str.replace(/\\r/g, '\r');
         str = str.replace(/\\\'/g, '\'');
         str = str.replace(/\\\"/g, '\"');
         str = str.replace(/\\\\/g, '\\');
         }
   }
   catch(e) {
      return e.message;
   }
   finally {
      return str;
   }
}


// try calling the web script a couple different ways
//try {

   //call the web service using SimpleHttpConnection
   // Does not work!
   //simplehttpresult = SimpleHttpConnection
    //           .getContentAsString("http://path/to/my/json.php");

   //call the web service synchronously via XMLHttpRequest
   XMLHttpRequest
                .open("GET", "http://path/to/my/json.php",
                         false, "user", "pass");
   XMLHttpRequest.send("");
   xmlhttpresult = convertjEsc2Char(XMLHttpRequest.getResponseText());
   XMLHttpRequest.close();
   
   var JSONobj = eval("(" + xmlhttpresult + ')' );

   ragsoc = JSONobj[1].answer_value;
   
//}catch(ex){error = String(ex)}

//uncomment to add to Freemarker model (if run as Web Script)
model.simplehttpresult = simplehttpresult;
model.xmlhttpresult = xmlhttpresult
model.error = error;
model.mystring = mystring;
</javascript>

The template

<html>
<head>
<!–<meta http-equiv="Content-type" content="text/html; charset=utf-8" />–>
</head>
<body>
<p>SimpleHttpConnection web service call result: ${simplehttpresult} <br /><br /> XMLHttpRequest web service call result: ${xmlhttpresult} <br /><br /> errors: ${error}</p>
<p>${mystring}</p>
</body>
</html>‍‍‍‍‍‍‍‍‍

The descriptor

<webscript>
   <shortname>wstest</shortname>
   <description>web service call test</description>
   <url>/xhrwstest</url>
   <authentication>user</authentication>
</webscript>‍‍‍‍‍‍