Saturday, May 20, 2017

Hikari Connection Pool Configuration

Here are some configuration information about Hikari Connection Pool Library


Spring Bean Configuration


<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:beans="http://www.springframework.org/schema/beans"
      xmlns:context="http://www.springframework.org/schema/context"
      xmlns:jdbc="http://www.springframework.org/schema/jdbc" xmlns:tx="http://www.springframework.org/schema/tx"
      xmlns:util="http://www.springframework.org/schema/util"
      xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
      http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
      http://www.springframework.org/schema/jdbc http://www.springframework.org/schema/jdbc/spring-jdbc.xsd
      http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx.xsd
      http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">

      <!-- Enable Annotation based Declarative Transaction Management -->
      <tx:annotation-driven proxy-target-class="true" transaction-manager="transactionManager" />

      <bean id="hikariConfig" class="com.zaxxer.hikari.HikariConfig">
            <property name="poolName" value="springHikariCP" />
            <property name="connectionTestQuery" value="SELECT 1" />
            <property name="dataSourceClassName" value="org.springframework.jdbc.datasource.DriverManagerDataSource" />
            <property name="connectionTimeout" value="${database.connection.connectionTimeout}" />
            <property name="validationTimeout" value="${database.connection.validationTimeout}" />
            <property name="idleTimeout" value="${database.connection.idleTimeout}" />
            <property name="maximumPoolSize" value="${database.connection.maximumPoolSize}" />
            <property name="minimumIdle" value="${database.connection.minimumIdle}" />
            <property name="leakDetectionThreshold" value="${database.connection.leakDetectionThreshold}" />
            <property name="maxLifetime" value="${database.connection.maxLifetime}" />
            <property name="autoCommit" value="false" />
            <property name="registerMbeans" value="true" />

            <property name="dataSourceProperties">
                  <props>
                        <prop key="url">${database.url}</prop>
                        <prop key="username">${database.username}</prop>
                        <prop key="password">${database.password}</prop>
                  </props>
            </property>
      </bean>

      <!-- HikariCP configuration -->
      <bean id="dataSource" class="com.zaxxer.hikari.HikariDataSource"
            destroy-method="close">
            <constructor-arg ref="hikariConfig" />
      </bean>

      <!-- JDBC Template -->
      <bean id="jdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
            <constructor-arg ref="dataSource" />
      </bean>

      <!-- Creating TransactionManager Bean, since JDBC we are creating of type DataSourceTransactionManager -->
      <bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
            <property name="dataSource" ref="dataSource" />
      </bean>

      <jdbc:initialize-database>
            <jdbc:script location="classpath:database/dml/parm_inserts.sql" />
      </jdbc:initialize-database>
</beans>



In properties file


database.connection.connectionTimeout=30000000
database.connection.validationTimeout=15000000
database.connection.idleTimeout=10000000
database.connection.maximumPoolSize=5
database.connection.minimumIdle=1
database.connection.leakDetectionThreshold=20000000
database.connection.maxLifetime=30000000


A utility class that logs the Hikari metrics to log file


@Service
public class DatabaseConnectionPoolMXBean {
      private static Logger logger = LoggerFactory.getLogger(DatabaseConnectionPoolMXBeam.class);

      public void logConnectionStatistics() {
            final MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
            ObjectName poolName = null;
            try {
                  poolName = new ObjectName("com.zaxxer.hikari:type=Pool (springHikariCP)");
            } catch (MalformedObjectNameException e) {
                  logger.warn("failed to create ObjectName: com.zaxxer.hikari:type=Pool (springHikariCP).", e);
            }
            final HikariPoolMXBean poolProxy = JMX.newMXBeanProxy(mBeanServer, poolName, HikariPoolMXBean.class);

            final int totalConnections = poolProxy.getTotalConnections();
            final int activeConnections = poolProxy.getActiveConnections();
            final int idleConnections = poolProxy.getIdleConnections();
            final int TheadsAwaitingConnections = poolProxy.getThreadsAwaitingConnection();

            final StringBuilder result = new StringBuilder("DatabaseConnectionPoolMXBeam, DatabasePool-totalConnections=")
                        .append(totalConnections).append(", DatabasePool-activeConnections=").append(activeConnections)
                        .append(", DatabasePool-idleConnections=").append(idleConnections)
                        .append(", DatabasePool-theadsAwaitingConnections=").append(TheadsAwaitingConnections);
            logger.info(result.toString());

      }


Friday, May 19, 2017

Correlation ID generator

When there is a need to follow the business logic route to collect all activities, a correlation id could be used as a key to bind all related log entries.


import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Random;

public class CorrelationHolder {
 private final static Random RANDOM = new Random();
 private final static int RANDOM_UPPER_LIMIT = 10000;
 private final static String DEFAULT = "DEFAULT";

 private final static ThreadLocal<String> CORRELATION_ID = new ThreadLocal<String>();

 private CorrelationHolder() {
  // The class is only supposed to be accessed in static
 }

 public static String getCorrelationId() {
  final String identifier = (String) CorrelationHolder.CORRELATION_ID.get();
  if (null == identifier) {
   try {
    CorrelationHolder.setCorrelationId(CorrelationHolder.createCorrelationId());
   } catch (Exception e) {
    CorrelationHolder.setCorrelationId(DEFAULT);
   }
  }
  return CorrelationHolder.CORRELATION_ID.get();
 }

 private static void setCorrelationId(final String identifier) {
  CorrelationHolder.CORRELATION_ID.set(identifier);
 }

 private static String createCorrelationId() throws Exception {
  String identifier = "";
  try {
   identifier = convert(InetAddress.getLocalHost().getHostName(), 12, false)
     + convert(String.valueOf(System.nanoTime()), 11, true)
     + convert(String.valueOf(RANDOM.nextInt(RANDOM_UPPER_LIMIT)), 6, true);
  } catch (UnknownHostException uhe) {
   throw new Exception("Error Creating CorrelationId" + uhe);
  }
  return identifier;
 }

 private static String convert(final String original, final int outputLength, final boolean isLeft) {
  if (original != null) {
   int len = original.length();
   if (len > outputLength) {
    if (isLeft) {
     return original.substring(0, outputLength);
    } else {
     return original.substring(len - outputLength);
    }

   } else if (len < outputLength) {
    final StringBuilder sBuilder = new StringBuilder();
    if (isLeft) {
     for (int i = 0; i < (outputLength - len); i++) {
      sBuilder.append("0");
     }
     sBuilder.append(original);
     return sBuilder.toString();
    } else {
     sBuilder.append(original);
     for (int i = 0; i < (outputLength - len); i++) {
      sBuilder.append("0");
     }
     return sBuilder.toString();
    }
   } else {
    return original;
   }
  }
  final StringBuilder sbuffer = new StringBuilder();
  for (int i = 0; i < outputLength; i++) {
   sbuffer.append("0");
  }
  return sbuffer.toString();
 }

 public static void main(String[] args) {
  System.out.println(CorrelationHolder.getCorrelationId());
 }
}



Thursday, May 18, 2017

Versioned JavaScript Importing

When a big JavaScript file is going to be included in a page, it should be cached at the client browser and also allow it to be updated immediately when there are changes to it.
ETag could be used to control the client side browser cache behaviors. If the JavaScript file is hosted in a  RestEasy supported web application, the following code may be used: 
public Response getJavaScripts(final Request request) throws ApiException {

    // Create cache control header
    final CacheControl cacheControl = new CacheControl();
    // Set max age to one day
    cacheControl.setMaxAge(86400);
    // Calculate the ETag on last modified date of user resource
    String eTag = this.applicationConfiguration.getPropertyValue(MY_JS_ETAG).orElse("v0.1");
    final EntityTag etag = new EntityTag(eTag);
    // Verify if it matched with etag available in http request
    final Response.ResponseBuilder responseBulider = request.evaluatePreconditions(etag);
    if (responseBulider == null) {
        // If rb is null then either it is first time request; or resource is modified,
        // get the updated representation and return with Etag attached to it
        final Object entity = this.myService.getJs();
        responseBulider = Response.ok(entity);
    }
    final Response response = responseBulider.cacheControl(cacheControl).tag(etag).build();
    return response;
}
However when it is tested in a browser, the latest changes may not be seen by page users even the Etag has been updated at the server side.
The issue is cause by the implementation of the browsers. Some browsers will not go back to server to check if there is a tag change or not. In order to have a reliable solution, more work has to be done.  One solution is using  an approach with two steps: downloading a short dynamic JavaScript first as bootstrap code, then a link in the bootstrap JavaScript downloads the more static and big JavaScript file.
A sample of the  bootstrap JavaScript:
/* dynamic.js  or use Ajax */
(function() {
    var c = document.createElement('script');
    c.type = 'text/javascript';
    c.async = true;
    c.src = ('https:' == document.location.protocol ? 'https://' : 'http://')
        + '<hostname:port>/myapp/js/v1/static.js?v=<version_number>';
    var s = document.getElementsByTagName('script')[0];
    s.parentNode.insertBefore(c, s);
})();

A link is going to be inserted to download this bootstrap JavaScript, there are two ways to keep this JavaScript from being cached at the client browser:
Attched a parameter (sessionId here) with a value it will be different when page loads:
<pre><script type="text/javascript" src="https://hostname:port/myapp/js/v1/dynamic.js?sessionId=<sessionId>">
</script></pre>

Or 
<script type="text/javascript" src="https://hostname:port/js/v1/dynamic.js"></script>
With following headers in response:
  • Expires: Sun, 19 Nov 1978 05:00:00 GMT
  • Last-Modified: Fri, 19 May 2017 08:01:46 GMT (the actual modification date)
  • Cache-Control: store, no-cache, must-revalidate, post-check=0, pre-check=0
Again, it RestEasy is used, here is the code to do this:
@Override
public Response getDynamicJavaScripts(final HttpServletRequest httpServletRequest ) throws ApiException {
    final String staticJs = getStaticJs(httpServletRequest);
    final Response.ResponseBuilder responseBulider = Response.ok().entity(staticJs);
    final Response response = getNoCacheResponse(responseBulider);
    return response;
}
private Response getNoCacheResponse(final Response.ResponseBuilder responseBulider) {
    final CacheControl cacheControl = new CacheControl();
    cacheControl.setMaxAge(0);
    cacheControl.setSMaxAge(0);
    cacheControl.setNoCache(true);
    cacheControl.setNoStore(true);
    cacheControl.setMustRevalidate(true);
    final Date expires = this.getUnixEpochStartDate();
    final Response response = responseBulider.cacheControl(cacheControl).expires(expires).header("post-check", 0).header("pre-check", 0).lastModified(new Date()).build();
    return response;
}
@Cacheable
private String getStaticJs(final HttpServletRequest request) {
    final String jsTemplate = "(function() { var c = document.createElement('script'); c.type = 'text/javascript'; c.async = true;" +
        "c.src = ('https:' == document.location.protocol ? 'https://' : 'http://') + '%s?v=%s';" +
        "var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(c, s); })();";
    final String jsServerUrl = new StringBuilder(request.getServerName()).append(":").append(request.getServerPort())
        .append(request.getContextPath()).append(request.getServletPath()).append(pathInfo).toString();
    final String version = this.applicationConfiguration.getPropertyValue(MY_JS_VERSION).orElse("V0.1");
    final String finalJs = String.format(jsTemplate, jsServerUrl, version);    
    return finalJs;
}
@Cacheable
private Date getUnixEpochStartDate() {
    final SimpleDateFormat isoFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
    isoFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
    Date date;
    try {
        date = isoFormat.parse("1970-01-01T00:00:00");
    } catch (ParseException e) {
        date = new Date();
        logger.warn("date passing error.", e);
    }
    return date;
}

Another useful feature from this two steps solution is that the dynamic JavaScript bootstrap could be used as a switch to turn on the functions provided by the static JavaScript file. If an empty page returned, the client page does not need to do any changes to turn off the function the static JavaScripts file provides.
It also could be used as a throttling mechanism. If bootstrap returns the static JavaScript file link in 50% of the responses, the load to the server that hosts the static JavaScript file will reduce 50%.
Usually a flag is required to notify the server side that the functions at client side has been turned off when data has been posted back to backend services.
The bootstrap JavaScript:
/* myjs_status_code could be used to indicate that the function switch status. For pages use basic JavaScript functions. */
var _myjs_status_code = '204';
/* sets the myjs_status_code hidden field after the page loaded if it exists. */
(function() {
    function setSessionId() {
            var element = document.getElementById('myjs_status_code');
            if (typeof (element) != 'undefined' && element != null) {
                element.value = _myjs_session_id;
            }
    }
    window.addEventListener('load', setSessionId, false);
    setSessionId(); /* in case the load event already passed. */
})();
/* the section below will not be returned if the static function should be turned off. */
(function() {
    var c = document.createElement('script');
    c.type = 'text/javascript';
    c.async = true;
    c.src = ('https:' == document.location.protocol ? 'https://' : 'http://')
        + '<hostname:port>/static.js?v=<version_number>';
    var s = document.getElementsByTagName('script')[0];
    s.parentNode.insertBefore(c, s);
})();



http://localhost:8080/app/api/RestEndpoint/getMethod/extraPathInfo
request.getProtocol(): HTTP/1.1
 contextPath: /app
servletPath: /api
pathInfo: /RestEndpoint/getMethod/extraPathInfo
requestUri = request.getContextPath() + request.getServletPath() + pathInfo







see more info here:
Drupal 6 does this (which works in every browser known by me):
  • Expires: Sun, 19 Nov 1978 05:00:00 GMT
  • Last-Modified: Fri, 12 Jun 2009 08:01:46 GMT (the actual modification date)
  • Cache-Control: store, no-cache, must-revalidate, post-check=0, pre-check=0
No pragma header in this instance. I'm not sure why your example doesn't work, it might be the negative timestamps, this works on ~250.000 Drupal sites :)
        
To force the browser to recheck your page with the server, the simplest solution is to add max-age=, must-revalidate=trueto the Cache-Control header. This will let your browser keep a copy of the page in its cache, but compare it with the server's version by sending the contents of Last-Modified or ETag, as you wanted.