Update_UMLS.gradle

/*
****************************
* UMLS Vocabulary update   *
* Version 0.0.7            *
****************************
*/

group 'com.sci.vocabulary'
version '1.0-SNAPSHOT'

apply plugin: 'java'

apply plugin: 'groovy'

sourceCompatibility = 1.8

buildscript {
    repositories {
        mavenLocal()
        mavenCentral()
    }
    dependencies {
        classpath 'org.apache.poi:poi:3.9'
        classpath 'org.apache.poi:poi-ooxml:3.9'
        classpath 'org.apache.httpcomponents:httpclient:4.5.3'
        classpath 'org.codehaus.groovy.modules.http-builder:http-builder:0.5.2'
        classpath 'com.squareup.okhttp3:okhttp:3.5.0'

        classpath 'commons-io:commons-io:2.5'
        classpath 'org.apache.httpcomponents:httpclient:4.5.3'
        classpath 'org.apache.httpcomponents:httpcore:4.4.6'
        classpath 'commons-httpclient:commons-httpclient:3.1'
        classpath 'commons-io:commons-io:2.5'

        classpath 'org.apache.commons:commons-lang3:3.5'
        classpath 'com.google.guava:guava:21.0'

        classpath group: 'commons-io', name: 'commons-io', version: '2.5'
        classpath group: 'org.apache.commons', name: 'commons-compress', version: '1.14'
    }
}
/*
repositories {
    mavenLocal()
    mavenCentral()
}
dependencies {
    classpath 'org.apache.poi:poi:3.9'
    classpath 'org.apache.poi:poi-ooxml:3.9'
}

*/

import groovy.io.FileType
import org.apache.poi.ss.usermodel.Cell
import org.apache.poi.ss.usermodel.Row
import org.apache.poi.ss.usermodel.Workbook
import org.apache.poi.ss.usermodel.WorkbookFactory
import org.apache.poi.ss.usermodel.Sheet

/*
task compileOne(type: JavaCompile) {
    source = sourceSets.main.java.srcDir
    include 'inc.odysseus.utils/StringBuilderWrapper.java'
    classpath = sourceSets.main.output.classesDir
    destinationDir = sourceSets.main.output.classesDir
}
compileOne.options.compilerArgs = ["-sourcepath", "${projectDir}/src/main/java"]

*/
/**
 * JAVA-CODE SECTION.
 * 
 * Temporary java code are located here. If future it's need to move it toi separate java-module.
 */
import com.google.common.base.Joiner;
import com.google.common.io.Files;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.io.FileUtils
import org.apache.http.HttpException;
import org.apache.http.HttpResponse;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.NameValuePair;
import org.apache.http.Header;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.cookie.Cookie;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HttpContext;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.http.util.EntityUtils;
import org.apache.compress.*;
import org.apache.http.*;

import java.io.*;
import java.net.URISyntaxException
import java.nio.charset.StandardCharsets
import java.nio.file.StandardCopyOption
import java.nio.file.StandardOpenOption
import java.nio.file.DirectoryStream
import java.util.*
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.usermodel.*;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.File;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import java.util.stream.Collectors
import java.util.zip.GZIPInputStream;

/**
 * Created by Sanders on 5/23/2017.
 */

public class IllegalUpdateStateException extends Exception {
    public IllegalUpdateStateException(String message) {
        super(message);
    }
    public IllegalUpdateStateException(Throwable cause) {
        super(cause);
    }
    public IllegalUpdateStateException(String message, Throwable cause) {
        super(message, cause);
    }
}

public class DownloadResourceHelper {
    /**
     * Names of using cookies.
     *
     * COOKIE_JSESSIONID - session identifier on UMLS login service;
     * COOKIE_CASTGC - client authentication cookie for UMLS login service;
     * COOKIE_MOD_AUTH_CAS - client ticket-authentication cookie for UMLS download service.
     */
    public static final String COOKIE_JSESSIONID = "JSESSIONID";
    public static final String COOKIE_CASTGC = "CASTGC";
    public static final String COOKIE_MOD_AUTH_CAS = "MOD_AUTH_CAS";
    public static final int NUMBER_OF_ATTEMPTS = 3;

    private CloseableHttpClient httpClient;
    private File downloadPath;
    /**
     * Presumably "Login Ticket" parameter.
     * Used on user form during authentication, generated by UMLS login service when new session is created.
     */
    private String ltParam;

    /**
     * Class-level cookie store
     */
    private List<Cookie> localCookieStore;

    /**
     * Apache CookieStore for all cookies.
     * Isn't used because of using class-level cookie store.
     */
//    private CookieStore cookieStore;

    /**
     * Instance of Download Helper.
     */
    private static DownloadResourceHelper singleIntance;

    /**
     * Contructor
     */
    private DownloadResourceHelper() {
        this(null);
    }

    private DownloadResourceHelper(File downloadPath) {
        if (downloadPath == null || !downloadPath.exists()) {
            // Prepare temporary storage for content
            this.downloadPath = Files.createTempDir();
            this.tempDir.deleteOnExit();
        } else
            this.downloadPath = downloadPath;

        this.localCookieStore = new ArrayList<Cookie>();

        // Create HttpClient
        this.httpClient = HttpClients.custom()
                //.setProxy(new HttpHost("127.0.0.1", 8888))
                .setRedirectStrategy(new LaxRedirectStrategy()) // adds HTTP REDIRECT support to GET and POST methods
//                .addInterceptorLast(
//                    new HttpRequestInterceptor() {
//                        public void process(HttpRequest httpRequest, HttpContext httpContext) throws HttpException, IOException {
//                            /**
//                             * Add common request headers
//                             */
//                            httpRequest.addHeader("Connection", "keep-alive");
//                            httpRequest.addHeader("Upgrade-Insecure-Requests", "1");
//                            httpRequest.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36");
//                            httpRequest.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
//                            httpRequest.addHeader("Accept-Encoding", "gzip, deflate, sdch, br");
//                            httpRequest.addHeader("Accept-Language", "en-US,en;q=0.8,ru;q=0.6,uk;q=0.4");
//                            httpRequest.addHeader("Cache-Control", "no-cache");
//                            httpRequest.addHeader("Pragma", "no-cache");
//                        }
//                    }
//                )
                .addInterceptorLast(
                    new HttpRequestInterceptor() {
                        public void process(HttpRequest httpRequest, HttpContext httpContext) throws HttpException, IOException {
                            System.out.println("\tREQUEST headers:");
                            for (Header h : httpRequest.getAllHeaders()) {
                                System.out.println(String.format("\t\t%s: %s", h.getName(), h.getValue()));
                            }
                        }
                    }
                )
                .addInterceptorLast(
                    new HttpResponseInterceptor() {
                        public void process(HttpResponse httpResponse, HttpContext httpContext) throws HttpException, IOException {
                            System.out.println("\tRESPONSE headers:");
                            for (Header h : httpResponse.getAllHeaders()) {
                                System.out.println(String.format("\t\t%s: %s", h.getName(), h.getValue()));
                            }
                        }
                    }
                )
                .build();
    }

    /**
     * Extract header value from response
     *
     * @param response
     * @param name
     * @return
     */
    private String getResponseHeaderValue(HttpResponse response, String name) {
        Header header = response.getLastHeader(name);
        if (header == null) {
            return "";
        } else {
            return header.getValue();
        }
    }

    /**
     * Convert array to List
     *
     * @param objects
     * @return
     */
    private <T> List<T> getListFromArray(T[] objects) {
        List<T> result = new ArrayList<T>();
        for (T c: objects) {
            if (c != null) {
                result.add(c);
            }
        }
        return result;
    }

    /**
     * Return all saved client's cookies.
     *
     * @return
     */
    private List<Cookie> getAllStoredCookies() {
        List<Cookie> ckList = new ArrayList<Cookie>();
        Iterator<Cookie> it = this.localCookieStore.iterator();
        while (it.hasNext()) {
            ckList.add(it.next());
        }
        return ckList;
    }

    /**
     * Get client cookie from local storage.
     *
     * @param name
     * @return
     */
    private Cookie getClientCookie(String name) {
        Cookie cookie = null;
        Iterator<Cookie> it = this.localCookieStore.iterator();
        while (it.hasNext()) {
            Cookie c = it.next();
            if (c.getName().equalsIgnoreCase(name)) {
                cookie = c;
                break;
            }
        }
        return cookie;
    }

    /**
     * Class-level Cookie Storage.
     *
     * @param request
     */
    private void saveClientCookie(HttpResponse request) {
        Header cookieHeader = request.getLastHeader("Set-Cookie");
        if (cookieHeader == null)
            return;
        String nameValue = cookieHeader.getValue().substring(0, cookieHeader.getValue().indexOf(";"));
        Cookie newCookie = new BasicClientCookie(nameValue.split("=")[0], nameValue.split("=")[1]);

        Iterator<Cookie> it = this.localCookieStore.iterator();
        while (it.hasNext()) {
            Cookie c = it.next();
            if (c.getName().equalsIgnoreCase(newCookie.getName()))
                return;
        }
        this.localCookieStore.add(newCookie);
    }

    /**
     * Add multiple cookies to response.
     *
     * @param request
     * @param cookies
     * @return
     */
    private HttpRequest setRequestCookies(HttpRequest request, List<Cookie> cookies) {
        if (cookies == null || cookies.isEmpty())
            return request;

        StringBuilder sb = new StringBuilder();
        for (Cookie c: cookies) {
            if (c != null) {
                if (sb.toString().length() != 0)
                    sb.append(";");
                sb.append(String.format("%s=%s", c.getName(), c.getValue()));
            }
        }
        if (sb.length() > 0)
            request.addHeader("Cookie", sb.toString());
        return request;
    }

    /**
     * Convert parameter's map to name-value format, for convenient usage for Apache's classes.
     *
     * @param params
     * @return
     */
    private List<NameValuePair> toMapNameValuePairs(Map<String, String> params) {
        List<NameValuePair> nvpList = new ArrayList<NameValuePair>();
        Iterator<Map.Entry<String, String>> it = params.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<String, String> entry = it.next();
            String name = entry.getKey();
            String value = entry.getValue();
            if (StringUtils.isNotBlank(name) && StringUtils.isNoneBlank(value))
                nvpList.add(new BasicNameValuePair(name, value));
        }
        return nvpList;
    }

    /**
     * Prepare GET request.
     *
     * @param url
     * @return
     * @throws IOException
     * @throws URISyntaxException
     */
    private HttpGet prepareGet(String url) throws IOException, URISyntaxException {
        return prepareGet(url, null, null);
    }

    /**
     * Prepare GET request with adding multiple cookies.
     *
     * @param url
     * @param cookies
     * @return
     * @throws IOException
     * @throws URISyntaxException
     */
    private HttpGet prepareGet(String url, List<Cookie> cookies) throws IOException, URISyntaxException {
        return prepareGet(url, null, cookies);
    }

    /**
     * Prepare parametrised GET request with adding multiple cookies.
     *
     * @param url
     * @param params
     * @param cookies
     * @return
     * @throws IOException
     * @throws URISyntaxException
     */
    private HttpGet prepareGet(String url, Map<String, String> params, List<Cookie> cookies) throws IOException, URISyntaxException {
        if (params != null && !params.isEmpty()) {
            URIBuilder uriBuilder = new URIBuilder(url);
            List<NameValuePair> pairList = toMapNameValuePairs(params);
            uriBuilder.addParameters(pairList);
            url = uriBuilder.toString();
        }
        HttpGet request = new HttpGet(url);
        setRequestCookies(request, cookies);
        return request;
    }

    /**
     * Prepare parametrised POST request
     *
     * @param url
     * @param params
     * @return
     * @throws IOException
     * @throws URISyntaxException
     */
    private HttpPost preparePost(String url, Map<String, String> params) throws IOException, URISyntaxException {
        return preparePost(url, params, null);
    }

    /**
     * Prepare parametrised POST request with adding multiple cookies.
     *
     * @param url
     * @param params
     * @param cookies
     * @return
     * @throws IOException
     * @throws URISyntaxException
     */
    private HttpPost preparePost(String url, Map<String, String> params, List<Cookie> cookies) throws IOException, URISyntaxException {
        HttpPost request = new HttpPost(url);
        if (params != null && !params.isEmpty()) {
            List<NameValuePair> pairList = toMapNameValuePairs(params);
            request.setEntity(new UrlEncodedFormEntity(pairList));
        }
        setRequestCookies(request, cookies);
        return request;
    }

    /**
     * Get instance
     *
     * @return
     */
    public static DownloadResourceHelper getDownloadResourceHelper(File downloadPath) {
        if (singleIntance == null) {
            singleIntance = new DownloadResourceHelper(downloadPath);
        }
        return singleIntance;
    }

    public String repeatableDownloadResourceUmls(String loginUrl, String userName, String password, String fileUrl, String fileName, String packageDescription) throws IllegalUpdateStateException {
        /**
         * Try three times redownload if error occurs
         */
        return downloadResourceUmls(loginUrl, userName, password, fileUrl, fileName, packageDescription, true, NUMBER_OF_ATTEMPTS);
    }

    public String downloadResourceUmls(String loginUrl, String userName, String password, String fileUrl, String fileName, String packageDescription) throws IllegalUpdateStateException {
        return downloadResourceUmls(loginUrl, userName, password, fileUrl, fileName, packageDescription, false, 0);
    }

    // Just download UMLS routine
    private String downloadUmls(String fileUrl, String fileName, packageDescription, boolean isTryRedownload, int numberOfTries) throws IllegalUpdateStateException {
        CloseableHttpResponse response = null;
        OutputStream outputStream = null;
        try {
            List<Cookie> cookies = new ArrayList<>();
            cookies.add(getClientCookie(COOKIE_MOD_AUTH_CAS));
            HttpGet request = prepareGet(fileUrl, cookies);
            response = httpClient.execute(request);

            int responseCode = response.getStatusLine().getStatusCode();
            if (responseCode != HttpStatus.SC_OK) {
                numberOfTries--;
                if (isTryRedownload && numberOfTries >= 0) {
                    try {
                        if (response != null) {
                            response.close();
                        }
                    } catch (IOException ioe) {}
                    System.out.println(String.format("Attention!\nThere is some problem of content downloading: %s.\nTry again, attempt %d of %d....", response.getStatusLine(), NUMBER_OF_ATTEMPTS - numberOfTries, NUMBER_OF_ATTEMPTS));
                    return downloadUmls(fileUrl, fileName, packageDescription, isTryRedownload, numberOfTries);
                } else {
                    throw new IllegalUpdateStateException(new HttpException(String.format("Unable to download %s\n%s", packageDescription, response.getStatusLine())));
                }
            } else if (getResponseHeaderValue(response, "Content-Type").indexOf("zip") < 0) {
                throw new IllegalUpdateStateException(new HttpException(String.format("%s is not a ZIP archive", packageDescription)));
            }

            // Eval downloaded file name
            String contentDisposition = getResponseHeaderValue(response, "Content-Disposition");
            if (contentDisposition.length() > 0)
                fileName = contentDisposition.substring(contentDisposition.lastIndexOf("filename") + 9).replace("\"", "").replace(";", "");
            String filePath = String.format("%s%s%s", this.downloadPath.getPath(), File.separator, fileName);

            // Start action checkpoint
            long timeStart = System.currentTimeMillis()
            String contentLength = getResponseHeaderValue(response,"Content-Length");
            System.out.println(new StringBuilderWrapper(String.format("Downloading of '%s' file with %,d bytes ...", fileName, contentLength)));

            // Save file to stream
            outputStream = new FileOutputStream(new File(filePath));
            IOUtils.copyLarge(response.getEntity().getContent(), outputStream);
            outputStream.flush();

            // Finish action checkpoint
            long timeFinish = System.currentTimeMillis()
            // Measure time execution
            long timeElapsed = (timeFinish - timeStart) / 1000
            System.out.println(new StringBuilderWrapper(String.format("Download completted in %,d seconds.", timeElapsed)));

        } catch (Exception e) {
            throw new IllegalUpdateStateException(e);
        } finally {
            if (response != null)
                response.close();
            if (outputStream != null)
                outputStream.close();
        }
        return fileName;
    }

    // Login and download resource
    public String downloadResourceUmls(String loginUrl, String userName, String password, String fileUrl, String fileName, String packageDescription, boolean isTryRedownload, int numberOfTries) throws IllegalUpdateStateException {
        CloseableHttpResponse response = null;
        // REQUEST 1
        // Just to get first JSESSION cookie
        try {
            HttpGet request = prepareGet(loginUrl);
            response = httpClient.execute(request);

            // Remember dynamically generated presumably "Login Ticket" form parameter
            String responseBody = EntityUtils.toString(response.getEntity()).trim();
            Pattern ltPattern = Pattern.compile("name=\"lt\" value=\"(\\w+)\"");
            Matcher m = ltPattern.matcher(responseBody);
            while (m.find()) {
                this.ltParam = m.group(1);
            }
        } catch (Exception e) {
            throw new IllegalUpdateStateException(e);
        } finally {
            if (response != null)
                response.close();
        }

        // REQUEST 2
        // Login
        System.out.println "${new StringBuilderWrapper("Login UMLS service...")}"
        try {
            Map<String, String> params = new HashMap<String, String>();
            params.put("username", userName);
            params.put("password", password);
            params.put("lt", this.ltParam);
            params.put("_eventId", "submit");
            params.put("submit", "Sign In");
            List<Cookie> cookies = new ArrayList<>();
            cookies.add(getClientCookie(COOKIE_JSESSIONID));
            HttpPost request = preparePost(loginUrl, params, cookies);
            request.addHeader("Content-Type", "application/x-www-form-urlencoded");
            response = httpClient.execute(request);

        } catch (Exception e) {
            throw new IllegalUpdateStateException(e);
        } finally {
            if (response != null)
                response.close();
        }

        // REQUEST 3
        // Get ticket for requested resource
        try {
            List<Cookie> cookies = new ArrayList<>();
            cookies.add(getClientCookie(COOKIE_JSESSIONID));
            cookies.add(getClientCookie(COOKIE_CASTGC));
            HttpGet request = prepareGet(fileUrl, cookies);
            response = httpClient.execute(request);

            /**
             * Get URL for further processing
             */
            fileUrl = getResponseHeaderValue(response, "Location");
        } catch (Exception e) {
            throw new IllegalUpdateStateException(e);
        } finally {
            if (response != null)
                response.close();
        }

        // REQUEST 4
        // Authorize on download service
        try {
            HttpGet request = prepareGet(fileUrl);
            response = httpClient.execute(request);

            /**
             * Get URL for further processing
             */
            fileUrl = getResponseHeaderValue(response,"Location");
        } catch (Exception e) {
            throw new IllegalUpdateStateException(e);
        } finally {
            if (response != null)
                response.close();
        }

        // REQUEST 5
        // Download resource
        return downloadUmls(fileUrl, fileName, packageDescription, isTryRedownload, numberOfTries);
    }

    public boolean loginLoinc(String loginUrl, String userName, String password) throws IllegalUpdateStateException {
        CloseableHttpResponse response = null;
        // Login LOINC services
        try {
            Map<String, String> params = new HashMap<String, String>();
            params.put("log", userName);
            params.put("pwd", password);
            params.put("wp-submit", "Log In");
            HttpPost request = preparePost(loginUrl, params);
            response = httpClient.execute(request);

            return (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK);
        } catch (Exception e) {
            throw new IllegalUpdateStateException(e);
        } finally {
            if (response != null)
                response.close();
        }
    }

    public String downloadResourceLoinc(String fileUrl, String fileName, String packageDescription) throws IllegalUpdateStateException {
        return downloadResourceLoinc(fileUrl, fileName, packageDescription, false, 0);
    }

    public String repeatableDownloadResourceLoinc(String fileUrl, String fileName, String packageDescription) throws IllegalUpdateStateException {
        /**
         * Try three times redownload if error occurs
         */
        return downloadResourceLoinc(fileUrl, fileName, packageDescription, true, NUMBER_OF_ATTEMPTS);
    }

    public String repeatableDownloadResourceUmlsFullRedirects(String fileUrl, String userName, String password, String fileName, String packageDescription) throws IllegalUpdateStateException {
        /**
         * Try three times redownload if error occurs
         */
        return downloadResourceUmlsFullRedirects(fileUrl, userName, password, fileName, packageDescription,true, NUMBER_OF_ATTEMPTS);
    }

    public String downloadResourceUmlsFullRedirects(String fileUrl,String userName, String password, String fileName, String packageDescription, boolean isTryRedownload, int numberOfTries) throws IllegalUpdateStateException {
        CloseableHttpResponse response = null;
        OutputStream outputStream = null;
        // Get JSESSION cookie
        HttpGet getRequest = null;
        try {
            System.out.println("\nStep 1. Get url: " + fileUrl);
            getRequest = prepareGet(fileUrl);

            System.out.println("!!!!! Request headers:");
            List<Header> lHeader = Arrays.asList(getRequest.getAllHeaders());
            for (Header h: lHeader) {
                System.out.println(String.format("\t%s: %s", h.getName(), h.getValue()));
            }
            response = httpClient.execute(getRequest);
            
            // Buffer response content
            BufferedHttpEntity bufEntity = new BufferedHttpEntity(response.getEntity());
            StringWriter writer = new StringWriter();
            IOUtils.copy(bufEntity.getContent(), writer, StandardCharsets.UTF_8);
            String responseBody = writer.toString().trim();

            System.out.println("\n!!!!! Response:");
            System.out.println("\tresponse.getStatusLine():" + response.getStatusLine());
            System.out.println("\tcontent-type:" + getResponseHeaderValue(response, "Content-Type"));
            System.out.println("\tlocation:" + getResponseHeaderValue(response, "Location"));
            System.out.println("\tset-cookie:");
            List<Header> listHeaders = Arrays.asList(response.getHeaders("Set-Cookie"));
            for (Header h: listHeaders) {
                System.out.println("\t\t" + h.getValue());
            }
            System.out.println("\tcontent-disposition:" + getResponseHeaderValue(response, "Content-Disposition"));
            System.out.println("\tresponseBody:" + responseBody);

            // Remember dynamically generated presumably "Login Ticket" form parameter
            Pattern ltPattern = Pattern.compile("name=\"lt\" value=\"(\\w+)\"");
            Matcher m = ltPattern.matcher(responseBody);
            while (m.find()) {
                this.ltParam = m.group(1);
            }
        } catch (Exception e) {
            getRequest.abort();
            throw new IllegalUpdateStateException(e);

        } finally {
            if (response != null)
                response.close();
        }

        // Login to UMLS and download package
        HttpPost postRequest = null;
        System.out.println("Login UMLS service...");
        try {
            Map<String, String> params = new HashMap<String, String>();
            params.put("username", userName);
            params.put("password", password);
            params.put("lt", this.ltParam);
            params.put("_eventId", "submit");
            params.put("submit", "Sign In");
            //List<Cookie> cookies = new ArrayList<>();
//            cookies.addAll(getAllStoredCookies());
            //cookies.add(getClientCookie(COOKIE_JSESSIONID));
            System.out.println("\nStep 2. Get url: " + fileUrl);
            postRequest = preparePost(fileUrl, params);
            postRequest.addHeader("Content-Type", "application/x-www-form-urlencoded");

            System.out.println("!!!!! Request headers:");
            List<Header> lHeader = Arrays.asList(postRequest.getAllHeaders());
            for (Header h: lHeader) {
                System.out.println(String.format("\t%s: %s", h.getName(), h.getValue()));
            }
            response = httpClient.execute(postRequest);

            System.out.println("\n!!!!! Response:");
            System.out.println("\tresponse.getStatusLine():" + response.getStatusLine());
            System.out.println("\tcontent-type:" + getResponseHeaderValue(response, "Content-Type"));
            System.out.println("\tlocation:" + getResponseHeaderValue(response, "Location"));
            System.out.println("\tset-cookie:");
            List<Header> listHeaders = Arrays.asList(response.getHeaders("Set-Cookie"));
            for (Header h: listHeaders) {
                System.out.println("\t\t" + h.getValue());
            }
            System.out.println("\tcontent-disposition:" + getResponseHeaderValue(response, "Content-Disposition"));

            int responseCode = response.getStatusLine().getStatusCode();
            if (responseCode != HttpStatus.SC_OK) {
                numberOfTries--;
                if (isTryRedownload && numberOfTries >= 0) {
                    if (response != null)
                        try {
                            response.close();
                        } catch (IOException ioe) {}
                    System.out.println(String.format("Attention!\nThere is some problem of content downloading: %s.\nTry again, attempt %d of %d....", response.getStatusLine(), NUMBER_OF_ATTEMPTS - numberOfTries, NUMBER_OF_ATTEMPTS));
                    return downloadResourceUmlsFullRedirects(fileUrl, userName, password, fileName, packageDescription, isTryRedownload, numberOfTries);
                } else {
                    throw new HttpException(String.format("Unable to download %s\n%s", packageDescription, response.getStatusLine()));
                }
            } else if (getResponseHeaderValue(response, "Content-Type").indexOf("zip") < 0) {
                throw new HttpException(String.format("%s is not a ZIP archive", packageDescription));
            }

            // Eval downloaded file name
            String contentDisposition = getResponseHeaderValue(response, "Content-Disposition");
            if (contentDisposition.length() > 0)
                fileName = contentDisposition.substring(contentDisposition.lastIndexOf("filename") + 9).replace("\"", "").replace(";", "");
            String filePath = String.format("%s%s%s", this.downloadPath.getPath(), File.separator, fileName);

            // Start action checkpoint
            long timeStart = System.currentTimeMillis()
            String contentLength = getResponseHeaderValue(response,"Content-Length");
            System.out.println(new StringBuilderWrapper(String.format("Downloading of '%s' file with %,d bytes ...", fileName, contentLength)));

            outputStream = new FileOutputStream(new File(filePath));
            IOUtils.copyLarge(response.getEntity().getContent(), outputStream);
            outputStream.flush();

            // Finish action checkpoint
            long timeFinish = System.currentTimeMillis()
            // Measure time execution
            long timeElapsed = (timeFinish - timeStart) / 1000
            System.out.println(new StringBuilderWrapper(String.format("Download completted in %,d seconds.", timeElapsed)));

        } catch (Exception e) {
            postRequest.abort();
            throw new IllegalUpdateStateException(e);

        } finally {
            if (response != null)
                response.close();
        }
        return fileName;
    }

    public String downloadResourceLoinc(String fileUrl, String fileName, String packageDescription, boolean isTryRedownload, int numberOfTries) throws IllegalUpdateStateException {
        OutputStream outputStream = null;
        CloseableHttpResponse response = null;
        try {
            Map<String, String> params = new HashMap<String, String>();
            params.put("tc_accepted", "1");
            params.put("tc_submit", "Download");
            List<Cookie> cookies = getAllStoredCookies();
            HttpPost request = preparePost(fileUrl, params, cookies);
            response = httpClient.execute(request);

            int responseCode = response.getStatusLine().getStatusCode();
            if (responseCode != HttpStatus.SC_OK) {
                numberOfTries--;
                if (isTryRedownload && numberOfTries > 0) {
                    try {
                        if (response != null) {
                            response.close();
                        }
                    } catch (IOException ioe) {}
                    System.out.println(String.format("Attention!\nThere is some problem of content downloading: %s.\nTry again, attempt %d of %d....", response.getStatusLine(), NUMBER_OF_ATTEMPTS - numberOfTries, NUMBER_OF_ATTEMPTS));
                    return downloadResourceLoinc(fileUrl, fileName, packageDescription, isTryRedownload, numberOfTries);
                } else {
                    throw new IllegalUpdateStateException(new HttpException(String.format("Unable to download %s\n%s", packageDescription, response.getStatusLine())));
                }
            } else if (getResponseHeaderValue(response, "Content-Type").indexOf("zip") < 0) {
                throw new IllegalUpdateStateException(new HttpException(String.format("%s is not a ZIP archive", packageDescription)));
            }

            // Eval downloaded file name
            String contentDisposition = getResponseHeaderValue(response, "Content-Disposition");
            if (contentDisposition.length() > 0) {
                fileName = contentDisposition.substring(contentDisposition.lastIndexOf("filename") + 9).replace("\"", "").replace(";", "");
            }
            String filePath = String.format("%s%s%s", this.downloadPath.getPath(), File.separator, fileName);

            // Start action checkpoint
            long timeStart = System.currentTimeMillis()
            String contentLength = getResponseHeaderValue(response,"Content-Length");
            System.out.println(new StringBuilderWrapper(String.format("Downloading of '%s' file with %,d bytes ...", fileName, contentLength)));

            // Save file to stream
            outputStream = new FileOutputStream(new File(filePath));
            IOUtils.copyLarge(response.getEntity().getContent(), outputStream);
            outputStream.flush();

            // Finish action checkpoint
            long timeFinish = System.currentTimeMillis()
            // Measure time execution
            long timeElapsed = (timeFinish - timeStart) / 1000
            System.out.println(new StringBuilderWrapper(String.format("Download completted in %,d seconds.", timeElapsed)));

        } catch (Exception e) {
            throw new IllegalUpdateStateException(e);
        } finally {
            if (response != null)
                response.close();
            if (outputStream != null)
                outputStream.close();
        }
        return fileName;
    }
}

/**
 * Created by Sanders on 5/26/2017.
 */
/**
 * Auxiliary classes. TODO: Further should be used java-defined and compiled only
 */
public class StringBuilderWrapper implements java.lang.Appendable {
    private static final String IDENT_CHAR = " " *4
    private StringBuilder wrapped

    public StringBuilderWrapper() {
        wrapped = new StringBuilder()
    }
    public StringBuilderWrapper(String value) {
        this()
        wrapped.append(IDENT_CHAR).append(value)
    }

    public static StringBuilderWrapper getBuilder() {
        return new StringBuilderWrapper()
    }

    public static String readFromFile(File file) {
        if (file == null)
            return ''
        def builder = getBuilder()
        def br = new BufferedReader(new FileReader(file))
        def strLine
        while ((strLine = br.readLine()) != null) {
                builder.append('\n').append(strLine)
        }
        return builder.toString()
    }

    @Override
    Appendable append(CharSequence csq) throws IOException {
        wrapped.append(IDENT_CHAR).append(csq)
        return this
    }
    @Override
    Appendable append(CharSequence csq, int start, int end) throws IOException {
        wrapped.append(IDENT_CHAR).append(csq, start, end)
        return this
    }
    @Override
    Appendable append(char c) throws IOException {
        wrapped.append(IDENT_CHAR).append(c)
        return this
    }
    @Override
    String toString() {
        return wrapped.toString()
    }
}

public class DataProcessor {
    
    public static void downloadPackage(File downloadPath) {
        if (downloadPath == null || !downloadPath.exists()) {
            // Prepare temporary storage for content
            downloadPath = Files.createTempDir();
            downloadPath.deleteOnExit();
        } else
            downloadPath= downloadPath;

        DownloadResourceHelper downloadHelper = DownloadResourceHelper.getDownloadResourceHelper(downloadPath);

        def timeStart = System.currentTimeMillis();
        
        String packageDescription = getPropertyByName("downloadUpdatePack.umlsFull.description");
        String fileUrl = getPropertyByName("downloadUpdatePack.umlsFull.fileUrl");
        String userName = getPropertyByName("downloadUpdatePack.umls.username");
        String password = getPropertyByName("downloadUpdatePack.umls.password");
        String defaultFileName = getPropertyByName("downloadUpdatePack.umlsFull.fileName");
        System.out.println("Downloading " + packageDescription);
        String downloadedFileName = downloadHelper.repeatableDownloadResourceUmlsFullRedirects(fileUrl, userName, password, defaultFileName, packageDescription);
        System.out.println("Downloaded file: " + downloadedFileName);
        System.out.println("Downloaded complete.\nUpdate packages are saved to the: " + downloadHelper.getDownloadPath());

        // Finish action checkpoint
        long timeFinish = System.currentTimeMillis();
        long timeElapsed = (timeFinish - timeStart) / 1000;
        System.out.println("Time elapsed: " + timeElapsed + " seconds.");

        System.out.println("*** " + packageDescription + " action done ***");
    }

    /**
     * Extract archives
     * @param is
     * @param extractFile
     * @param inputFileSize
     * @throws IOException
     */
    private static void writeToFile(InputStream is, File extractFile, long inputFileSize) throws IOException {
        System.out.print(String.format("\tprocessing file: %s ", extractFile.getName()));
        byte[] buffer = new byte[8192];
        int count;
        OutputStream outputStream = null;
        long progress = 0, totalProgress = 0;
        byte percentProgress;
        byte decimalCount = 0;
        try {
            outputStream = new BufferedOutputStream(new FileOutputStream(extractFile));
            while ((count = is.read(buffer, 0, buffer.length)) != -1) {
                outputStream.write(buffer, 0, count);
                progress += count;
                totalProgress += count;
                percentProgress = Long.valueOf(Math.round(((double) progress / inputFileSize) * 100)).byteValue();
                if (percentProgress / 10 > decimalCount) {
                    decimalCount ++;
                    System.out.print(".");
                }
            }
            outputStream.flush();
        } finally {
            if (outputStream != null)
                outputStream.close();
        }
        float displaySize;
        String sizeUnit;
        if (totalProgress < 1024) {
            sizeUnit = "B";
            displaySize = Double.valueOf(totalProgress).floatValue();
        } else if ((totalProgress / 1024) < 1024) {
            sizeUnit = "KB";
            displaySize = Double.valueOf((double)(totalProgress / 1024)).floatValue();
        } else if ((totalProgress / 1024 / 1024) < 1024) {
            sizeUnit = "MB";
            displaySize = Double.valueOf((double)(totalProgress / 1024 / 1024)).floatValue();
        } else {
            sizeUnit = "GB";
            displaySize = Double.valueOf((double)(totalProgress / 1024 / 1024 / 1024)).floatValue();
        }
        System.out.println(String.format(" %.0f %s is done", displaySize, sizeUnit));
    }

    public static <I extends InputStream> boolean extractAction(I archiveInputStream, File archiveFile, File destPath) {
        System.out.println(String.format("\nExtracting file: %s is extracted to the: %s", archiveFile.getPath(), destPath.getPath()));
        try {
            File fileDir;
            if (archiveInputStream instanceof  ArchiveInputStream) {
                ArchiveInputStream ais = (ArchiveInputStream) archiveInputStream;
                ArchiveEntry entry = ais.getNextEntry();
                while (entry != null) {
                    if (entry.isDirectory()) {
                        fileDir = new File(destPath, entry.getName());
                        fileDir.mkdirs();
                    } else {
                        fileDir = new File(destPath, entry.getName());
                        writeToFile(archiveInputStream, fileDir, entry.getSize());
                    }
                    entry = ais.getNextEntry();
                }
            } else {
                String fileName = GzipUtils.getUncompressedFilename(archiveFile.getName());
                fileDir = new File(destPath, fileName);
                writeToFile(archiveInputStream, fileDir, archiveFile.length());
            }
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        } finally {
            if (archiveInputStream != null)
                try {
                    archiveInputStream.close();
                } catch (IOException ioe) {}
        }
        return true;
    }

    public static boolean extractArchive(File archiveFile, File destPath) {
        if (archiveFile == null || !archiveFile.exists() || !destPath.exists())
            return false;

        boolean result = false;
        InputStream archiveInputStream = null;
        try {
            switch (archiveFile.getName().substring(archiveFile.getName().lastIndexOf(".")+ 1)) {
                case "nlm":
                case "zip":
                    archiveInputStream = new ZipArchiveInputStream(
                            new BufferedInputStream(new FileInputStream(archiveFile))
                    );
                    result = extractAction(archiveInputStream, archiveFile, destPath);
                    break;
                case "gz":
                    if (true) {
                        archiveInputStream = new GZIPInputStream(new BufferedInputStream(new FileInputStream(archiveFile)));
                        result = extractAction(archiveInputStream, archiveFile, destPath);
                    }
                    break;
            }
        } catch (IOException ioe) {
            ioe.printStackTrace();
        } finally {
            if (archiveInputStream != null)
                try {
                    archiveInputStream.close();
                } catch (IOException e) {}
        }
        return result;
    }

    /**
     * Extract files 
     * @param workDir
     * @return
     */
    public static void extract(File workDir) {
        extract(workDir, null);
    }

    public static void extract(File workDir, List<String> fileNames) {
        System.out.println(String.format("*** Extract executed, workDir: %s, fileNames: %s", workDir.getPath(), fileNames));

        if (fileNames == null)
            fileNames = new ArrayList<>();

        DirectoryStream<Path> stream = null;
        try {
            stream = Files.newDirectoryStream(Paths.get(workDir.getPath()));
            for (Path path: stream) {
                if (path.toFile().isDirectory())
                    extract(path.toFile(), fileNames);
                else {
                    String fName = path.getFileName().toString().toLowerCase();
                    if (fName.endsWith("zip") || fName.endsWith("gz") || fName.endsWith("nlm"))
                        fileNames.add(path.toAbsolutePath().toString());
                }
            }
        } finally {
            if (stream != null)
                stream.close();
        }

        System.out.println("\t!!! Found files to extract:");
        for (String filename: fileNames) {
            System.out.println(String.format("\t%s", filename));
            if (!extractArchive(Paths.get(filename).toFile(), workDir)) {
                System.out.println(String.format("Cannot extract file: %s", filename.getPath()));
            }
        }
        System.out.println("***");

//        for (File f: workDir.listFiles(
//                new FileFilter() {
//                    @Override
//                    public boolean accept(File pathname) {
//                        return pathname.getName().toLowerCase().endsWith("zip") ||
//                                pathname.getName().toLowerCase().endsWith("gz") ||
//                                pathname.getName().toLowerCase().endsWith("nlm");
//                    }
//                }))
//        {
//
//            if (!extractArchive(f, workDir)) {
//                System.out.println(String.format("Cannot extract file: %s", f.getPath()));
//            }
//        }
    }
    
    public static void concatenate(File workDir) {
        File fPath = workDir;

        List<String> fileMaskList = Arrays.asList("MRCONSO", "MRHIER", "MRSAT");
        LinkedHashMap<String, TreeSet<File>> orderedFilesParts = new LinkedHashMap<>();
        for (String fileMask: fileMaskList) {
            TreeSet<File> orderedFiles = new TreeSet<>();
            for (File f : fPath.listFiles(new FileFilter() {
                @Override
                public boolean accept(File pathName) {
                    String name = pathName.getName().toLowerCase();
                    return name.indexOf(fileMask.toLowerCase()) >= 0 &&
                            !name.endsWith("zip") && !name.endsWith("gz") &&
                            (name.length() - name.replaceAll("\\.", "").length()) > 1;
                }
            })) {
                orderedFiles.add(f);
            }
            String fileName = orderedFiles.first().getName();
            System.out.println(String.format("!!! fileName 1: %s", fileName));
            fileName = fileName.substring(0, fileName.lastIndexOf("."));
            System.out.println(String.format("!!! fileName 2: %s", fileName));
            orderedFilesParts.put(fileName, orderedFiles);
        }

        Iterator<Map.Entry<String, TreeSet<File>>> it = orderedFilesParts.entrySet().iterator();
        while(it.hasNext()) {
            Map.Entry<String, TreeSet<File>> e = it.next();
            String targetFile = new File(fPath, e.getKey()).getPath();
            System.out.print(String.format("Concatenating all parts of the %s file ...\n", targetFile));
            OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(targetFile));
            try {
                Iterator<File> fit = e.getValue().iterator();
                while (fit.hasNext()) {
                    File partFile = fit.next();
                    BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(partFile));
                    try {
                        byte[] buff = new byte[8192];
                        int bytesRead;
                        while((bytesRead = inputStream.read(buff, 0, buff.length)) >= 0) {
                            outputStream.write(buff, 0, bytesRead);
                        }
                    } finally {
                        if (inputStream != null)
                            inputStream.close();
                    }
                }
            } finally {
                if (outputStream != null) {
                    outputStream.flush();
                    outputStream.close();
                }
            }
            System.out.println(" done");
        }
    }
}

/**
 * JAVA-CODE SECTION
 */

// Load UMLS properties
Properties props = new Properties()
props.load(new FileInputStream("${project.getBuildFile().toString().replace('.gradle', '.properties')}"))
props.each {prop ->
    project.ext.set(prop.key, prop.value)
}

//def tempDir = File.createTempDir()
//tempDir.deleteOnExit()
def tempDir = new File("C:\\Users\\Sanders\\AppData\\Local\\Temp\\groovy-generated-5406289380510812740-tmpdir");

/**
 * Just renew user schema and prepare it for further vocabulary update
 */
task fastRecreateSchema(description: project.properties['fastRecreate.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*       '${description}' action started...     *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['fastRecreate.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['fastRecreate.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['fastRecreate.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['fastRecreate.streamErrFileName']}"

    def db = [un     : project.properties['db.userName'],
              pw     : project.properties['db.password'],
              tns    : project.properties['db.tnsName'],
              sqlfile: sqlFile,
              logfile: logFile]
    def sOut = new StringBuilderWrapper(), sErr = new StringBuilderWrapper()
    def sqlPlus = """sqlplus -L ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile}""".execute()
    sqlPlus.consumeProcessOutput(sOut, sErr)
    sqlPlus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    /**
     * Write to console.
     * Write compact SQLPlus's spool log
     */
    File lf = new File(logFile)
    if (lf.exists())
        System.out.println(StringBuilderWrapper.readFromFile(lf))

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sOut.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(sErr.toString())

    /* Check for errors */
    if (Boolean.valueOf(project.properties['stop.on.error'].toString()) && (sqlPlus.exitValue() != 0 || (sErr != null && sErr.toString().length() > 0))) {
        throw new GradleException(String.format(project.properties['error.message.with.code'], sqlPlus.exitValue(), name))
    }

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * This task will prepare user schema for vocabulary update routine.
 * It will create auxiliary tables - the "ICD10CM_TABLE","LOINC","MAP_TO","SOURCE_ORGANIZATION","LOINC_ANSWERS","LOINC_FORMS","LOINC_CLASS","CPT_MRSMAP","scccRefset_MapCorrOrFull_INT" in our case, for example.
 */
task createSourceTable(dependsOn: fastRecreateSchema, description: project.properties['createSourceTable.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*   '${description}' action started...   *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['createSourceTable.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['createSourceTable.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['createSourceTable.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['createSourceTable.streamErrFileName']}"

    def db = [un     : project.properties['db.userName'],
              pw     : project.properties['db.password'],
              tns    : project.properties['db.tnsName'],
              sqlfile: sqlFile,
              logfile: logFile]
    def sOut = new StringBuilderWrapper(), sErr = new StringBuilderWrapper()
    def sqlPlus = """sqlplus ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile}""".execute()
    sqlPlus.consumeProcessOutput(sOut, sErr)
    sqlPlus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    /**
     * Write to console.
     * Write compact SQLPlus's spool log
     */
    File lf = new File(logFile)
    if (lf.exists())
        System.out.println(StringBuilderWrapper.readFromFile(lf))

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sOut.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(sErr.toString())

    /* Check for errors */
    if (Boolean.valueOf(project.properties['stop.on.error'].toString()) && (sqlPlus.exitValue() != 0 || (sErr != null && sErr.toString().length() > 0))) {
        throw new GradleException(String.format(project.properties['error.message.with.code'], sqlPlus.exitValue(), name))
    }

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * Task which will download vocabulary update package from source host.
 * There are many URLs (in see the https://github.com/OHDSI/Vocabulary-v5.0/blob/master/LOINC/readme.md for details)
 */
task downloadUpdatePack(dependsOn: createSourceTable, description: project.properties['downloadUpdatePack.description']) << {
    println '\n\n*******************************************************'
    println '*                                                     *'
    println "*    '${description}' action started...     *"
    println '*                                                     *'
    println '*******************************************************'

//    /* Downloading file */
//    // Start action checkpoint
    def timeStart = System.currentTimeMillis()
//    DownloadResourceHelper downloadHelper = DownloadResourceHelper.getDownloadResourceHelper(tempDir);
//
//    /****************************************************
//     * Download 'UMLS' package
//     * **************************************************/
//    String packageDescription = project.properties['downloadUpdatePack.umlsFull.description']
//    String fileUrl = project.properties['downloadUpdatePack.umlsFull.fileUrl']
//    String userName = project.properties['downloadUpdatePack.umls.username']
//    String password = project.properties['downloadUpdatePack.umls.password']
//    String defaultFileName = project.properties['downloadUpdatePack.umlsFull.fileName']
//    println new StringBuilderWrapper("Downloading ${packageDescription}...")
//
//    String downloadedFileName = downloadHelper.repeatableDownloadResourceUmlsFullRedirects(fileUrl, userName, password, defaultFileName, packageDescription);

    String downloadedFileName = "umls-2017AA-full.zip";
    
    project.ext.set('downloadUpdatePack.umlsFull.fileName', downloadedFileName);
    println new StringBuilderWrapper("Downloaded complete.\nUpdate packages are saved to the: ${tempDir.getPath()}")

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * This task will extract files and prepare it for import
 */
task prepareFiles(dependsOn: downloadUpdatePack, description: project.properties['prepareFiles.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*     '${description}' action started...       *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()
    
    try {
        // Extract files
        DataProcessor.extract(tempDir);

        // Extract files again
        DataProcessor.extract(tempDir);

        // Concatenate part of files
        DataProcessor.concatenate(tempDir);

    } catch (Exception e) {
        e.printStackTrace();

        throw new GradleException(String.format(project.properties['error.message'], e.getMessage(), name))
    }

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * Task performed import CSV data to database
 */
task importData(dependsOn: prepareFiles, description: project.properties['importData.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*         '${description}' action started...          *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    /**
     * Filter import-specific parameters
     */
    Map<String, String> importParams = project.properties.findAll {
        it.getKey().startsWith("importData.") && it.getKey().toLowerCase().indexOf("description") < 0
    }
    /**
     * Group import-specific parameters
     */
    Map<String, String> importParamsGrouped = importParams.groupBy {it.getKey().substring(0, it.getKey().lastIndexOf("."))}

    /**
     * Process import
     */
    Iterator<Map.Entry<?, ?>> importIterator = importParamsGrouped.entrySet().iterator()
    while(importIterator.hasNext()) {
        Map.Entry<?, ?> importEntry = importIterator.next()
        String groupNameLabel = importEntry.getKey().replaceAll("\\.", " ").capitalize()
        Map<String, String> groupParams = importEntry.getValue()

        String controlFileName = groupParams.find {it.getKey().toLowerCase().indexOf("controlFileName".toLowerCase()) >= 0}.getValue()
        String dataFileName = new File(tempDir, groupParams.find {it.getKey().toLowerCase().indexOf("dataFileName".toLowerCase()) >= 0}.getValue()).getPath()
        String logFileName = groupParams.find {it.getKey().toLowerCase().indexOf("logFileName".toLowerCase()) >= 0}.getValue()
        String discardFileName = groupParams.find {it.getKey().toLowerCase().indexOf("discardFileName".toLowerCase()) >= 0}.getValue()
        String badFileName = groupParams.find {it.getKey().toLowerCase().indexOf("badFileName".toLowerCase()) >= 0}.getValue()
        String streamLogFileName = groupParams.find {it.getKey().toLowerCase().indexOf("streamLogFileName".toLowerCase()) >= 0}.getValue()
        String streamErrorFileName = groupParams.find {it.getKey().toLowerCase().indexOf("streamErrorFileName".toLowerCase()) >= 0}.getValue()

        println "\n${groupNameLabel}...\n"

        def bosOut = new ByteArrayOutputStream(), bosErr = new ByteArrayOutputStream()
        def sOut = new StringBuilderWrapper(), sErr = new StringBuilderWrapper()
        exec {
            executable "sqlldr"
            args "USERID=${project.properties['db.userName']}/${project.properties['db.password']}@${project.properties['db.tnsName']} " +
                    "CONTROL=${project.properties['basePath']}/${controlFileName} " +
                    "DATA=${dataFileName} " +
                    "LOG=${project.properties['logPath']}/${logFileName} " +
                    "DISCARD=${project.properties['logPath']}/${discardFileName} " +
                    "BAD=${project.properties['logPath']}/${badFileName}"
            standardOutput = bosOut
            errorOutput = bosErr
        }
        def bisOut = new ByteArrayInputStream(bosOut.toByteArray())
        bisOut.withReader { out->
            out.eachLine {
                sOut.append(it).append('\n')
            }
        }
        def bisErr = new ByteArrayInputStream(bosErr.toByteArray())
        bisErr.withReader { out ->
            out.eachLine {
                sErr.append(it).append('\n')
            }
        }

        /**
         *  Write to console.
         *  When import prefer to write stream log to console (it's shorter).
         */
        System.out.println(sOut.toString())

        // Write to logfile
        File f = new File("${project.properties['logPath']}/${streamLogFileName}")
        f.write(sOut.toString())

        // Write to error file
        f = new File("${project.properties['logPath']}/${streamErrorFileName}")
        f.write(sErr.toString())
    }

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * This task will run scripts which allows to see the vocabulary update results
 */
task checkUpdateResults(dependsOn: importData, description: project.properties['checkUpdateResults.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*  '${description}' action started...   *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['checkUpdateResults.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['checkUpdateResults.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['checkUpdateResults.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['checkUpdateResults.streamErrFileName']}"

    def db = [un     : project.properties['db.userName'],
              pw     : project.properties['db.password'],
              tns    : project.properties['db.tnsName'],
              sqlfile: sqlFile,
              logfile: logFile]
    def sOut = new StringBuilderWrapper(), sErr = new StringBuilderWrapper()
    def sqlPlus = """sqlplus ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile}""".execute()
    sqlPlus.consumeProcessOutput(sOut, sErr)
    sqlPlus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    /**
     * Write to console.
     * Write compact SQLPlus's spool log
     */
    File lf = new File(logFile)
    if (lf.exists())
        System.out.println(StringBuilderWrapper.readFromFile(lf))

    // Write to log file
    File f = new File(streamLogFile)
    f.write(sOut.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(sErr.toString())

    /* Check for errors */
    if (Boolean.valueOf(project.properties['stop.on.error'].toString()) && (sqlPlus.exitValue() != 0 || (sErr != null && sErr.toString().length() > 0))) {
        throw new GradleException(String.format(project.properties['error.message.with.code'], sqlPlus.exitValue(), name))
    }

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * Bootstrap action
 *
 * TODO: Needs to find solution to measure whole task chain execution.
 * TODO: This one via doFirst/doLast actions isn't working.
 */
task startUpdate(dependsOn: checkUpdateResults, description: project.properties['startUpdate.description']) {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*      '${description}' action started...       *"
    println '*                                             *'
    println '***********************************************'

    println "Updating of '${project.properties['vocabulary.display.name']}' vocabulary on the '${project.properties['db.userName']}@${project.properties['db.tnsName']}' schema...\n"

    project.ext['logPath'] = "${project.properties['basePath']}/${project.properties['baseLogPath']}"
    def folder = new File(project.properties['logPath'])
    if (!folder.exists()) {
        folder.mkdirs()
        println "Log directory is created: ${folder.getPath()}"
    } else {
        FileUtils.cleanDirectory(folder)
        println "Log directory is cleaned: ${folder.getPath()}"
    }

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    doLast {
        // Finish action checkpoint
        def timeFinish = System.currentTimeMillis()

        // Measure time execution
        def timeElapsed = (timeFinish - timeStart) / 1000
        println "\n\nTotal time elapsed: ${timeElapsed} seconds."
        println "Updating of '${project.properties['vocabulary.display.name']}' vocabulary on the '${project.properties['db.userName']}@${project.properties['db.tnsName']}' schema is completed successfully..."
        println '***********************************************'
        println '*                                             *'
        println "*      '${description}' action is done...       *"
        println '*                                             *'
        println '***********************************************\n'
    }
}