Syntax highlighter header

Showing posts with label Java. Show all posts
Showing posts with label Java. Show all posts

Thursday, 14 August 2025

Java 21 with Spring Security 6.5.2, Oauth2 and AWS Cognito Pool

Recently we were trying to migrate our spring boot application to Java 21 and latest spring boot version to for enhancing security of our application. We hit major road blocks related to working of JWT tokens. Here I am documenting the solution which worked for us. Major idea is similar to my previous post https://blog.bigdatawithjasvant.com/2023/08/spring-security-60-with-oauth2-and-aws.html

But there were some new challenges which we faced. Let us get started.

We were having some public apis which were open to all without any authentication and some apis were secured with Cognito pool JWT token. This application was different from the one described in my previous post. So we were not building on top of application described in my previous post. For configuring access to apis you need to create a configuration class called SecurityConfiguration like below:

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
import org.springframework.security.config.http.SessionCreationPolicy;
import org.springframework.security.oauth2.jwt.JwtDecoder;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;

import java.util.Arrays;

@Configuration
@EnableWebSecurity
@EnableMethodSecurity
public class SecurityConfig {

    @Autowired
    private JwtDecoder jwtDecoder;

    @Bean
    public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception {
        // @formatter:off
        http.cors(c-> {
            UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
            CorsConfiguration config = new CorsConfiguration();
            config.setAllowCredentials(true);
            config.addAllowedOriginPattern("*");
            config.addAllowedHeader("*");
            config.setAllowedMethods(Arrays.asList("OPTIONS", "GET", "POST", "PUT", "DELETE", "PATCH"));
            source.registerCorsConfiguration("/**", config);

            c.configurationSource(source);
        });
        http.csrf(AbstractHttpConfigurer::disable);

        http
                .authorizeHttpRequests((authorize) -> authorize
                        .requestMatchers("/","/api/public/**","/actuator/**").permitAll()
                        .anyRequest().authenticated()
                ).sessionManagement(sm-> sm.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
                .oauth2ResourceServer(rs->  rs.jwt(j-> j.decoder(jwtDecoder)));
        // @formatter:on
        return http.build();
    }

}

For decoding JWT tokens we need to use JwtDecoder. The JWT decoder is defined in another config class like this:

import com.nimbusds.jose.*;
import com.nimbusds.jose.proc.JWSKeySelector;
import com.nimbusds.jose.proc.SecurityContext;
import com.nimbusds.jwt.proc.DefaultJWTProcessor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.convert.converter.Converter;
import org.springframework.security.oauth2.jwt.JwtDecoder;
import org.springframework.security.oauth2.jwt.MappedJwtClaimSetConverter;
import org.springframework.security.oauth2.jwt.NimbusJwtDecoder;
import java.security.Key;
import java.util.*;

import com.nimbusds.jose.jwk.JWKSet;
import com.nimbusds.jose.jwk.RSAKey;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;

import java.text.ParseException;

@Configuration
public class JwtConfiguration {

    public static final String COGNITO_GROUPS = "cognito:groups";
    private static final String SPRING_AUTHORITIES = "scope";
    public static final String COGNITO_USERNAME = "username";
    private static final String SPRING_USER_NAME = "sub";

    @Value("${security.oauth2.resource.jwk.key-set-uri}")
    private String keySetUri;

    @Bean
    JwtDecoder jwtDecoder() throws ParseException {
        // Obtain the JWKS from the endpoint
        RestTemplate restTemplate = new RestTemplate();
        ResponseEntity<String> jwksResponse = restTemplate.getForEntity(keySetUri, String.class);
        String jwksJson = jwksResponse.getBody();

        JWKSet jwkSet = JWKSet.parse(jwksJson);

        DefaultJWTProcessor<SecurityContext> defaultJWTProcessor =  new DefaultJWTProcessor<>();

        defaultJWTProcessor.setJWSKeySelector(new JWSKeySelector<SecurityContext>() {
            @Override
            public List<? extends Key> selectJWSKeys(JWSHeader header, SecurityContext context) throws KeySourceException {
                RSAKey rsaKey = (RSAKey) jwkSet.getKeyByKeyId(header.getKeyID());
                try {
                    return new ArrayList<Key>(Arrays.asList(rsaKey.toPublicKey()));
                } catch (JOSEException e) {
                    e.printStackTrace();
                }
                return null;
            }
        });
        NimbusJwtDecoder jwtDecoder = new NimbusJwtDecoder(defaultJWTProcessor);

        Converter<Map<String, Object>, Map<String, Object>> claimSetConverter = MappedJwtClaimSetConverter
                .withDefaults(Collections.emptyMap());

        jwtDecoder.setClaimSetConverter( claims -> {
            claims = claimSetConverter.convert(claims);

            HashMap<String, Object> hashMap = new HashMap<>(claims);
            if (claims.containsKey(COGNITO_GROUPS))
                ((Map<String, Object>) hashMap).put(SPRING_AUTHORITIES, claims.get(COGNITO_GROUPS));
            if (claims.containsKey(COGNITO_USERNAME))
                ((Map<String, Object>) hashMap).put(SPRING_USER_NAME, claims.get(COGNITO_USERNAME));

            return hashMap;
        });
        return jwtDecoder;
    }
}

When I tried running this code I faced a class not found exception which took a lot of time to resolve. The Exception was:

[ main] o.s.boot.SpringApplication , 857 : Application run failed java.lang.ClassNotFoundException: org.springframework.security.oauth2.server.resource.authentication.DPoPAuthenticationProvider 
  at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) 
  at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) 
  at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:526) 
  ... 57 common frames omitted 
Wrapped by: java.lang.NoClassDefFoundError: org/springframework/security/oauth2/server/resource/authentication/DPoPAuthenticationProvider

The reason of the problem was mismatch between different versions of spring security libraries. The problem was solved when I used following versions of the dependencies:

	<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>3.5.3</version>
		<relativePath/> <!-- lookup parent from repository -->
	</parent>
        <dependencies>
		<dependency>
			<groupId>com.fasterxml.jackson.core</groupId>
			<artifactId>jackson-databind</artifactId>
			<version>2.15.2</version> <!-- match AWS SDK requirements -->
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.springframework.security/spring-security-jwt -->
		<dependency>
			<groupId>org.springframework.security</groupId>
			<artifactId>spring-security-jwt</artifactId>
			<version>1.1.1.RELEASE</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.springframework.security/spring-security-oauth2-client -->
		<dependency>
			<groupId>org.springframework.security</groupId>
			<artifactId>spring-security-oauth2-client</artifactId>
			<version>6.5.2</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.springframework.security/spring-security-config -->
		<dependency>
			<groupId>org.springframework.security</groupId>
			<artifactId>spring-security-config</artifactId>
			<version>6.5.2</version>
		</dependency>
		<dependency>
			<groupId>org.springframework.security</groupId>
			<artifactId>spring-security-oauth2-resource-server</artifactId>
			<version>6.5.2</version>
		</dependency>
		<dependency>
			<groupId>org.springframework.security</groupId>
			<artifactId>spring-security-oauth2-jose</artifactId>
			<version>6.5.2</version>
		</dependency>
		<dependency>
			<groupId>com.nimbusds</groupId>
			<artifactId>nimbus-jose-jwt</artifactId>
			<version>10.0.2</version>
		</dependency>
      </dependencies>

Please comment if you find it useful or need some more info.

Friday, 8 November 2024

NewRelic error with spring boot

 Recently we added two datasources to our application and our application worked fine without newRelic. But as soon as we added newRelic to our application it failed with a strange error:

2024-11-08 20:22:37 java.lang.ArrayIndexOutOfBoundsException: Index 0 out of bounds for length 0
2024-11-08 20:22:37     at org.jboss.jandex.Indexer.updateTypeTarget(Indexer.java:903)
2024-11-08 20:22:37     at org.jboss.jandex.Indexer.updateTypeTargets(Indexer.java:630)
2024-11-08 20:22:37     at org.jboss.jandex.Indexer.index(Indexer.java:1698)
2024-11-08 20:22:37     at org.hibernate.boot.archive.scan.spi.ClassFileArchiveEntryHandler.toClassDescriptor(ClassFileArchiveEntryHandler.java:64)
2024-11-08 20:22:37     at org.hibernate.boot.archive.scan.spi.ClassFileArchiveEntryHandler.handleEntry(ClassFileArchiveEntryHandler.java:52)
2024-11-08 20:22:37     at org.hibernate.boot.archive.internal.JarFileBasedArchiveDescriptor.visitArchive(JarFileBasedArchiveDescriptor.java:147)
2024-11-08 20:22:37     at org.hibernate.boot.archive.scan.spi.AbstractScannerImpl.scan(AbstractScannerImpl.java:48)
2024-11-08 20:22:37     at org.hibernate.boot.model.process.internal.ScanningCoordinator.coordinateScan(ScanningCoordinator.java:76)
2024-11-08 20:22:37     at org.hibernate.boot.model.process.spi.MetadataBuildingProcess.prepare(MetadataBuildingProcess.java:107)
2024-11-08 20:22:37     at org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl.<init>(EntityManagerFactoryBuilderImpl.java:269)
2024-11-08 20:22:37     at org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl.<init>(EntityManagerFactoryBuilderImpl.java:182)
2024-11-08 20:22:37     at org.springframework.orm.jpa.vendor.SpringHibernateJpaPersistenceProvider.createContainerEntityManagerFactory(SpringHibernateJpaPersistenceProvider.java:52)
2024-11-08 20:22:37     at org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean.createNativeEntityManagerFactory(LocalContainerEntityManagerFactoryBean.java:365)
2024-11-08 20:22:37     at org.springframework.orm.jpa.AbstractEntityManagerFactoryBean.buildNativeEntityManagerFactory(AbstractEntityManagerFactoryBean.java:409)
2024-11-08 20:22:37     at org.springframework.orm.jpa.AbstractEntityManagerFactoryBean.afterPropertiesSet(AbstractEntityManagerFactoryBean.java:396)
2024-11-08 20:22:37     at org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean.afterPropertiesSet(LocalContainerEntityManagerFactoryBean.java:341)
2024-11-08 20:22:37     at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1863)
2024-11-08 20:22:37     at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1800)

After struggling for a lot of time we figured out that one of the datasource was not having any entity defined. After defining a dummy entity for the datasource the error was gone away and the application started working with newRelic also.

Please do comment if you find it useful.

Wednesday, 7 February 2024

Creating a proxy for other service in Spring Boot

Recently we had a requirement to translate from HTTP headers to query parameter to expose a service to a client which can't send query parameter to the service. The client could send the information as HTTP header although.

We decided to build it in our spring boot application using FeignClient. We accepted API name as path variable so that same code can work for multiple APIs.

Following is the code our FeignClient interface which accept API name as a path variable:

import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;

@FeignClient(name = "genericClient", configuration = GenericClientConfiguration.class,
        url = "${base-url}" )
public interface GenericClient {
    String LOGIN_ID = "_loginid";
    String USER_TOKEN = "_token";

    @PostMapping(value = "/prefix/{apiName}", consumes = "application/json", produces = "application/json")
    ResponseEntity<String> callAPI( @PathVariable String apiName,
            @RequestParam(value = LOGIN_ID) String loginId,
            @RequestParam(value = USER_TOKEN) String userToken,
            @RequestBody String requestBody);
}

Following is the code for our controller class which receive the request from client. This controller catches errors returned by the server and passes them to the client.

import feign.FeignException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;

import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Optional;

@RestController
@Slf4j
@RequestMapping("/api/public")
public class ProxyController {

    @Autowired
    private GenericClient genericClient;

    @PostMapping(value = "/proxyPrefix/{api}", consumes = "application/json", produces = "application/json")
    public ResponseEntity<?> omsApiCall(@PathVariable String api,
                                           @RequestHeader String userName,
                                           @RequestHeader String userToken,
                                           @RequestBody String reqBody) {
        try {
            ResponseEntity<String> resp= genericClient.callAPI(api, userName, userToken, reqBody);
            log.info("Generic call successful API={}",api);
            return resp;
        } catch(FeignException.FeignClientException fce) {
            log.info("Generic call failed: API={}", api, fce);
            Optional<ByteBuffer> respBody = fce.responseBody();
            if(respBody.isPresent()) {
                String errorText = StandardCharsets.UTF_8.decode(respBody.get()).toString();
                return ResponseEntity.status(fce.status()).body(errorText);
            } else {
                return ResponseEntity.status(fce.status()).build();
            }
        } catch(FeignException.FeignServerException fse) {
            log.error("Generic call failed with server error for API={}", api, fse);
            Optional<ByteBuffer> respBody = fse.responseBody();
            if(respBody.isPresent()) {
                String errorText = StandardCharsets.UTF_8.decode(respBody.get()).toString();
                return ResponseEntity.status(fse.status()).body(errorText);
            } else {
                return ResponseEntity.status(fse.status()).build();
            }
        } catch(FeignException fe) {
            log.error("Generic call failed with unknown error for API={}", api, fe);
            Optional<ByteBuffer> respBody = fe.responseBody();
            if(respBody.isPresent()) {
                String errorText = StandardCharsets.UTF_8.decode(respBody.get()).toString();
                return ResponseEntity.status(fe.status()).body(errorText);
            } else {
                return ResponseEntity.status(fe.status()).build();
            }
        }
    }
}

Following is the code for our trusting Feign Client configuration which is needed incase you are connecting to a https url with self signed certificate:

import feign.Client;
import org.apache.http.ssl.SSLContexts;
import org.apache.http.ssl.TrustStrategy;
import org.springframework.context.annotation.Bean;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;

public class GenericClientConfiguration {

    @Bean
    public Client feignClient()
            throws KeyStoreException, NoSuchAlgorithmException, KeyManagementException {
        TrustStrategy acceptingTrustStrategy = (X509Certificate[] chain, String authType) -> true;
        HostnameVerifier hostnameVerifier = (s, sslSession) -> true;
        SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build();
        return new Client.Default(sslContext.getSocketFactory(),hostnameVerifier);
    }
}

Thursday, 24 August 2023

Using new HttpClient API for downloading file

Recently we were required to download a CSV file from our partner website in our Java application. So I explored new Java HttpClient API. The APIs are asynchronous which is good for scalability but confusing for the users. After struggling for a long time I was able to figure out how to download the file which was protected by a username and password. 

The code for downloading file is provided below, it also include a call to get just headers using HEAD call.

import java.io.IOException;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;

public class HttpClientMainClass {
    public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
        HttpClient httpClient = HttpClient.newBuilder().authenticator(new Authenticator() {
                    @Override
                    protected PasswordAuthentication getPasswordAuthentication() {
                        return new PasswordAuthentication("username","password".toCharArray());
                    }
                })
                .build();

        HttpRequest httpRequest2 = HttpRequest.newBuilder().method("HEAD", HttpRequest.BodyPublishers.noBody())
                .uri(new URI("http://example.com/data.csv"))
                .build();
        HttpResponse<Void> resp2 = httpClient.send(httpRequest2,
                HttpResponse.BodyHandlers.discarding());

        System.out.println(resp2);
        System.out.println(resp2.headers());

        HttpRequest httpRequest = HttpRequest.newBuilder().GET()
                .uri(new URI("http://example.com/data.csv"))
                .build();
        HttpResponse<Path> resp = httpClient.send(httpRequest,
                HttpResponse.BodyHandlers.ofFile(Path.of("/tmp","data.csv"),
                        StandardOpenOption.CREATE, StandardOpenOption.WRITE));

        if(resp.statusCode()==200) {
            System.out.println("File downloaded At : "+resp.body());
        }
    }
}

Hope it will be useful for the readers.

Ref: https://www.baeldung.com/java-9-http-client

Friday, 4 August 2023

Spring Security 6.0 with Oauth2 and AWS Cognito Pool

Recently we were migrating our old application from Spring Security 5 to Spring security 6. The major problem we faced was there was no documentation available on internet for Spring Security 6. After struggling for a long time we were able to move our old application to latest version of spring which is 6.0 . Here I am documenting our finding in hope that it will be helpful for the readers.

The first class you need to write is SecurityConfiguration. The content of file is provided below:

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
import org.springframework.security.config.http.SessionCreationPolicy;
import org.springframework.security.oauth2.jwt.JwtDecoder;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;

import java.util.Arrays;

@Configuration
@EnableWebSecurity
@EnableMethodSecurity
public class SecurityConfiguration {

    @Autowired
    private JwtDecoder jwtDecoder;

    @Bean
    public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception {
        // @formatter:off
        http.cors(c-> {
            UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
            CorsConfiguration config = new CorsConfiguration();
            config.setAllowCredentials(true);
            config.addAllowedOriginPattern("*");
            config.addAllowedHeader("*");
            config.setAllowedMethods(Arrays.asList("OPTIONS", "GET", "POST", "PUT", "DELETE", "PATCH"));
            source.registerCorsConfiguration("/**", config);

            c.configurationSource(source);
        });
        http.csrf(AbstractHttpConfigurer::disable);

        http
                .authorizeHttpRequests((authorize) -> authorize
                        .requestMatchers(AntPathRequestMatcher.antMatcher("/error")).permitAll()
                        .requestMatchers(AntPathRequestMatcher.antMatcher("/api/public/**")).permitAll()
                        .requestMatchers(AntPathRequestMatcher.antMatcher("/api/test/**")).permitAll()
                        .requestMatchers(AntPathRequestMatcher.antMatcher("/actuator/**")).permitAll()
                        .anyRequest().authenticated()
                ).sessionManagement(sm-> sm.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
                .oauth2ResourceServer(rs-> rs.jwt(j-> j.decoder(jwtDecoder)));
        // @formatter:on
        return http.build();
    }
}

In the above class I defined a CorsFilter to allow all domains. CSRF is disabled and few URLs were allowed without authentication. We need to use OAuth2 for integrating Cognito pool JWT tokens, for that we specified jwtDecoder. The JwtDecoder is defined in next class:

import com.nimbusds.jose.*;
import com.nimbusds.jose.proc.JWSKeySelector;
import com.nimbusds.jose.proc.SecurityContext;
import com.nimbusds.jwt.proc.DefaultJWTProcessor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.convert.converter.Converter;
import org.springframework.security.oauth2.jwt.JwtDecoder;
import org.springframework.security.oauth2.jwt.MappedJwtClaimSetConverter;
import org.springframework.security.oauth2.jwt.NimbusJwtDecoder;
import java.security.Key;
import java.util.*;

import com.nimbusds.jose.jwk.JWKSet;
import com.nimbusds.jose.jwk.RSAKey;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;

import java.text.ParseException;

@Configuration
public class JwtConfiguration {

    public static final String COGNITO_GROUPS = "cognito:groups";
    private static final String SPRING_AUTHORITIES = "scope";
    public static final String COGNITO_USERNAME = "username";
    private static final String SPRING_USER_NAME = "sub";

    @Value("${security.oauth2.resource.jwk.key-set-uri}")
    private String keySetUri;

    @Bean
    JwtDecoder jwtDecoder() throws ParseException {
        // Obtain the JWKS from the endpoint
        RestTemplate restTemplate = new RestTemplate();
        ResponseEntity<String> jwksResponse = restTemplate.getForEntity(keySetUri, String.class);
        String jwksJson = jwksResponse.getBody();

        JWKSet jwkSet = JWKSet.parse(jwksJson);

        DefaultJWTProcessor<SecurityContext> defaultJWTProcessor =  new DefaultJWTProcessor<>();

        defaultJWTProcessor.setJWSKeySelector(new JWSKeySelector<SecurityContext>() {
            @Override
            public List<? extends Key> selectJWSKeys(JWSHeader header, SecurityContext context) throws KeySourceException {
                RSAKey rsaKey = (RSAKey) jwkSet.getKeyByKeyId(header.getKeyID());
                try {
                    return new ArrayList<Key>(Arrays.asList(rsaKey.toPublicKey()));
                } catch (JOSEException e) {
                    e.printStackTrace();
                }
                return null;
            }
        });
        NimbusJwtDecoder jwtDecoder = new NimbusJwtDecoder(defaultJWTProcessor);

        Converter<Map<String, Object>, Map<String, Object>> claimSetConverter = MappedJwtClaimSetConverter
                .withDefaults(Collections.emptyMap());

        jwtDecoder.setClaimSetConverter( claims -> {
            claims = claimSetConverter.convert(claims);

            HashMap<String, Object> hashMap = new HashMap<>(claims);
            if (claims.containsKey(COGNITO_GROUPS))
                ((Map<String, Object>) hashMap).put(SPRING_AUTHORITIES, claims.get(COGNITO_GROUPS));
            if (claims.containsKey(COGNITO_USERNAME))
                ((Map<String, Object>) hashMap).put(SPRING_USER_NAME, claims.get(COGNITO_USERNAME));

            return hashMap;
        });
        return jwtDecoder;
    }
}

In the above class we are trying to decode the JWT token generated by Cognito pool which is passed to our API as bearer token. The JWT tokens are signed using a private key and validated using a public key. The public keys are provided as a JSON Web Key Set(JWKS) at the URL https://cognito-idp.<aws-region>.amazonaws.com/<cognito-pool-id>/.well-known/jwks.json by AWS. It contains public keys for validating tokens. There can be multiple keys in this set so you need to parse this JSON Web Key Set(JWKS) and pick the correct key as per header of JWT token.

After that you need to do some customization in claimSetConverter. The customization we required are that we want to use username of the cognito pool as user identification rather than UUID generated by cognito for the user. For doing that we overwritten value of "sub" with username received in claim. This value will be returned by principal.getName() when used in the code. 

The second customization we did was to overwrite "scope" with "cognito:groups" because we want to use congnito groups as authority for authenticating our API calls. Assume cognito group name is ROLE_ADMIN then the authority it will be mapped to will be SCOPE_ROLE_ADMIN and we can use @PreAuthorize("hasAuthority('SCOPE_ROLE_ADMIN')") for pre authorizing the API calls. One sample API is implemented below:

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.security.Principal;

@RestController
@RequestMapping("/api/users")
@RequiredArgsConstructor
@Validated
@Slf4j
public class UserNameTestController {
    @GetMapping("/userName")
    @PreAuthorize("hasAuthority('SCOPE_ROLE_ADMIN')")
    public  ResponseEntity<String>  getCurrentUserInfo(final Principal principal) {
        return ResponseEntity.status(200).body(principal.getName());
    }
}

The important maven dependencies used in this code are:

        <dependency>
            <groupId>org.springframework.security</groupId>
            <artifactId>spring-security-config</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.security</groupId>
            <artifactId>spring-security-web</artifactId>
        </dependency>
		<dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-security</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.security</groupId>
            <artifactId>spring-security-oauth2-resource-server</artifactId>
            <version>6.0.2</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.security</groupId>
            <artifactId>spring-security-oauth2-jose</artifactId>
            <version>6.0.5</version>
        </dependency>
        <dependency>
            <groupId>com.nimbusds</groupId>
            <artifactId>nimbus-jose-jwt</artifactId>
            <version>9.8.1</version>
        </dependency>

I hope this will be helpful for the readers. Please drop me a comment if you need some more information.

References:

https://medium.com/javarevisited/json-web-key-set-jwks-94dc26847a34 

Friday, 26 May 2023

Unable to connect to aurora DB from java 11

Recently we were trying to connect to 5.7.mysql_aurora.2.11.2 DB from Java 11 and were getting communication link failure in our Spring boot application. We were getting following error:
2023-05-26 21:20:43.826  INFO [,] 10788 --- [           main] com.zaxxer.hikari.HikariDataSource       : HikariPool-3 - Starting...
2023-05-26 21:20:45.756 ERROR [,] 10788 --- [           main] com.zaxxer.hikari.pool.HikariPool        : HikariPool-3 - Exception during pool initialization.

com.mysql.cj.jdbc.exceptions.CommunicationsException: Communications link failure

The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.
	at com.mysql.cj.jdbc.exceptions.SQLError.createCommunicationsException(SQLError.java:174) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping.translateException(SQLExceptionsMapping.java:64) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:836) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:456) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:246) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:198) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.zaxxer.hikari.util.DriverDataSource.getConnection(DriverDataSource.java:138) ~[HikariCP-3.4.5.jar:na]
	at com.zaxxer.hikari.pool.PoolBase.newConnection(PoolBase.java:358) ~[HikariCP-3.4.5.jar:na]
	at com.zaxxer.hikari.pool.PoolBase.newPoolEntry(PoolBase.java:206) ~[HikariCP-3.4.5.jar:na]
	at com.zaxxer.hikari.pool.HikariPool.createPoolEntry(HikariPool.java:477) ~[HikariCP-3.4.5.jar:na]
	at com.zaxxer.hikari.pool.HikariPool.checkFailFast(HikariPool.java:560) ~[HikariCP-3.4.5.jar:na]
	at com.zaxxer.hikari.pool.HikariPool.<init>(HikariPool.java:115) ~[HikariCP-3.4.5.jar:na]
	at com.zaxxer.hikari.HikariDataSource.getConnection(HikariDataSource.java:112) ~[HikariCP-3.4.5.jar:na]
	at org.hibernate.engine.jdbc.connections.internal.DatasourceConnectionProviderImpl.getConnection(DatasourceConnectionProviderImpl.java:122) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.engine.jdbc.env.internal.JdbcEnvironmentInitiator$ConnectionProviderJdbcConnectionAccess.obtainConnection(JdbcEnvironmentInitiator.java:180) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.engine.jdbc.env.internal.JdbcEnvironmentInitiator.initiateService(JdbcEnvironmentInitiator.java:68) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.engine.jdbc.env.internal.JdbcEnvironmentInitiator.initiateService(JdbcEnvironmentInitiator.java:35) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.boot.registry.internal.StandardServiceRegistryImpl.initiateService(StandardServiceRegistryImpl.java:101) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.service.internal.AbstractServiceRegistryImpl.createService(AbstractServiceRegistryImpl.java:263) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.service.internal.AbstractServiceRegistryImpl.initializeService(AbstractServiceRegistryImpl.java:237) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.service.internal.AbstractServiceRegistryImpl.getService(AbstractServiceRegistryImpl.java:214) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.id.factory.internal.DefaultIdentifierGeneratorFactory.injectServices(DefaultIdentifierGeneratorFactory.java:152) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.service.internal.AbstractServiceRegistryImpl.injectDependencies(AbstractServiceRegistryImpl.java:286) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.service.internal.AbstractServiceRegistryImpl.initializeService(AbstractServiceRegistryImpl.java:243) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.service.internal.AbstractServiceRegistryImpl.getService(AbstractServiceRegistryImpl.java:214) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.boot.internal.InFlightMetadataCollectorImpl.<init>(InFlightMetadataCollectorImpl.java:176) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.boot.model.process.spi.MetadataBuildingProcess.complete(MetadataBuildingProcess.java:127) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl.metadata(EntityManagerFactoryBuilderImpl.java:1224) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl.build(EntityManagerFactoryBuilderImpl.java:1255) ~[hibernate-core-5.4.25.Final.jar:5.4.25.Final]
	at org.springframework.orm.jpa.vendor.SpringHibernateJpaPersistenceProvider.createContainerEntityManagerFactory(SpringHibernateJpaPersistenceProvider.java:58) ~[spring-orm-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean.createNativeEntityManagerFactory(LocalContainerEntityManagerFactoryBean.java:365) ~[spring-orm-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.orm.jpa.AbstractEntityManagerFactoryBean.buildNativeEntityManagerFactory(AbstractEntityManagerFactoryBean.java:391) ~[spring-orm-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.orm.jpa.AbstractEntityManagerFactoryBean.afterPropertiesSet(AbstractEntityManagerFactoryBean.java:378) ~[spring-orm-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean.afterPropertiesSet(LocalContainerEntityManagerFactoryBean.java:341) ~[spring-orm-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1853) ~[spring-beans-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1790) ~[spring-beans-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:594) ~[spring-beans-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:516) ~[spring-beans-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:324) ~[spring-beans-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234) ~[spring-beans-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:322) ~[spring-beans-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202) ~[spring-beans-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.context.support.AbstractApplicationContext.getBean(AbstractApplicationContext.java:1109) ~[spring-context-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:869) ~[spring-context-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:551) ~[spring-context-5.2.12.RELEASE.jar:5.2.12.RELEASE]
	at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.refresh(ServletWebServerApplicationContext.java:143) ~[spring-boot-2.3.7.RELEASE.jar:2.3.7.RELEASE]
	at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:758) ~[spring-boot-2.3.7.RELEASE.jar:2.3.7.RELEASE]
	at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:750) ~[spring-boot-2.3.7.RELEASE.jar:2.3.7.RELEASE]
	at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:405) ~[spring-boot-2.3.7.RELEASE.jar:2.3.7.RELEASE]
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:315) ~[spring-boot-2.3.7.RELEASE.jar:2.3.7.RELEASE]
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1237) ~[spring-boot-2.3.7.RELEASE.jar:2.3.7.RELEASE]
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1226) ~[spring-boot-2.3.7.RELEASE.jar:2.3.7.RELEASE]
	at com.influencers.shared.InfluencersBackofficeApplication.main(InfluencersBackofficeApplication.java:12) ~[classes/:na]
Caused by: com.mysql.cj.exceptions.CJCommunicationsException: Communications link failure

The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.
	at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[na:na]
	at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[na:na]
	at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[na:na]
	at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490) ~[na:na]
	at com.mysql.cj.exceptions.ExceptionFactory.createException(ExceptionFactory.java:61) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.exceptions.ExceptionFactory.createException(ExceptionFactory.java:105) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.exceptions.ExceptionFactory.createException(ExceptionFactory.java:151) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.exceptions.ExceptionFactory.createCommunicationsException(ExceptionFactory.java:167) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.protocol.a.NativeProtocol.negotiateSSLConnection(NativeProtocol.java:340) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.protocol.a.NativeAuthenticationProvider.connect(NativeAuthenticationProvider.java:167) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.protocol.a.NativeProtocol.connect(NativeProtocol.java:1348) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.NativeSession.connect(NativeSession.java:157) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:956) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:826) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	... 50 common frames omitted
Caused by: javax.net.ssl.SSLHandshakeException: No appropriate protocol (protocol is disabled or cipher suites are inappropriate)
	at java.base/sun.security.ssl.HandshakeContext.<init>(HandshakeContext.java:170) ~[na:na]
	at java.base/sun.security.ssl.ClientHandshakeContext.<init>(ClientHandshakeContext.java:103) ~[na:na]
	at java.base/sun.security.ssl.TransportContext.kickstart(TransportContext.java:238) ~[na:na]
	at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:394) ~[na:na]
	at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:373) ~[na:na]
	at com.mysql.cj.protocol.ExportControlled.performTlsHandshake(ExportControlled.java:317) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.protocol.StandardSocketFactory.performTlsHandshake(StandardSocketFactory.java:188) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.protocol.a.NativeSocketConnection.performTlsHandshake(NativeSocketConnection.java:97) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	at com.mysql.cj.protocol.a.NativeProtocol.negotiateSSLConnection(NativeProtocol.java:331) ~[mysql-connector-java-8.0.22.jar:8.0.22]
	... 55 common frames omitted
This error is occurs because 5.7.mysql_aurora.2.11.2 DB is not supporting SSL protocols which are enabled by Java 11. We tried passing connection parameters to JDBC connector to enable TLSv1.2 in the server but it did not help. We had to modify java.security file to remove TLSv1 and TLSv1.1 from list of disabled algorithms.

We were using AWS ECS for deploying our application so we were creating a docker image. We modified the Dockerfile to copy modified java.security file in the docker image. Following is the content of the Dockerfile:
FROM eclipse-temurin:11
ARG JAR_FILE=target/*.jar
COPY ${JAR_FILE} app.jar
COPY java.security ${JAVA_HOME}/conf/security/java.security
ENTRYPOINT ["java","-jar","/app.jar"]
This modification of java.security file solved our problem. Other methods of enabling TLSv1.2 did not work. We were able to connect MySQL 5.7 instance in docker without modifying java.security file but not with auora database.

The modified java.security file can be downloaded from https://www.bigdatawithjasvant.com/blogdata/00/0003/java.security

Saturday, 26 June 2021

When to use ForkJoinPool vs ExecutorService?

With introduction of ForkJoinPool in Java people started getting confused weather to use ForkJoinPool or ExecutorService. In this post I am going to discuss which thread pool use in which case.

ForkJoinPool is designed to be used for CPU intensive workloads. The default number of threads in ForkJoinPool is equal to number of CPUs on the system. If any threads goes into waiting state due to calling join() on some other ForkJoinTask an new compensatory thread is started to utilize all CPUs of the system. ForkJoinPool has a common pool which can be get by calling ForkJoinPool.commonPool() static method. The aim of this design is to use only single ForkJoinPool in the system with number of threads being equal to number of processors on the system. It can utilize full computation capacity of the system if all ForkJoinTasks are doing computation intensive activities.

But in real life scenario tasks are a mix of CPU and IO intensive tasks. IO intensive task are a bad choice for a ForkJoinPool. You should use Executor service for doing IO intensive tasks. In ExecutorService you can set number of threads according to IO capacity of your system instead of CPU capacity of your system.

If you want to call an IO intensive operation from a ForkJoinTask then you should create a class which implement ForkJoinPool.ManagedBlocker interface and do IO intensive operation in block() method. You need to call your ForkJoinPool.ManagedBlocker implementation using static method ForkJoinPool.managedBlock(). This method creates a compensatory threads before calling block() method. block() method is supposed to do IO operation and store result in some instance variable. After calling ForkJoinPool.managedBlock() you are supposed to call your business method to get result of IO operation. This way you can mix CPU intensive operations with IO intensive operations. A classic example is WebCrawler where you fetch pages from internet which is an IO intensive operation and after that you need to parse the HTML page to extract links which is a CPU intensive operation. 

I have not implemented a full WebCrawler but a sample code where I fetch web pages using an ExecutorService with 10 threads. I am using common pool of ForkJoinPool for submitting ForkJoinTasks. My ForkJoinTask submits the page fetch request to ExecutorService and wait for result using ForkJoinPool.managedBlock() static method. After getting the page it calculates SHA-256 sum for the content of the page and stores it in a ConcurrentHashMap. This way we can make full use of CPU capacity of the system and IO capacity of the system.

The sample code is:


import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future;
import java.util.concurrent.RecursiveTask;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
public class ForkJoinPoolTest {
	
	public static class FetchPage implements ForkJoinPool.ManagedBlocker {
		
		private String url;
		private ExecutorService executorSerivce;
		private byte[] pageBytes;
		
		private static ConcurrentHashMap<String,byte[]> pagesMap = new ConcurrentHashMap<>();
		
		public FetchPage(String url, ExecutorService executorSerivce) {
			this.url = url;
			this.executorSerivce = executorSerivce;
		}

		@Override
		public boolean block() throws InterruptedException {
			if((pageBytes= pagesMap.get(url))!=null) {
				return true;
			}
			Callable<byte[]> callable= new Callable<byte[]>() {
				public byte[] call() throws Exception {
					CloseableHttpClient client = HttpClients.createDefault();
					HttpGet request = new HttpGet(url);
					CloseableHttpResponse response = client.execute(request);
					return EntityUtils.toByteArray(response.getEntity());
				}
			};
			Future<byte[]> future = executorSerivce.submit(callable);
			try {
				pageBytes = future.get();
			} catch (InterruptedException | ExecutionException e) {
				pageBytes=null;
			}
			return true;
		}

		@Override
		public boolean isReleasable() {
			if(pageBytes!=null) {
				return true;
			}
			return false;
		}
		
		public byte[] getPage() {
			return pageBytes;
		}
		
	}
	
	private static ConcurrentHashMap<String, String> hashPageMap = new ConcurrentHashMap<>();
	
	public static class MyRecursiveTask extends RecursiveTask<String> {
		
		private String url;
		private ExecutorService executorSerivce;
		public MyRecursiveTask(String url, ExecutorService executorSerivce) {
			this.url = url;
			this.executorSerivce = executorSerivce;
		}

		protected String compute() {
			try {
				FetchPage fp = new FetchPage(url,executorSerivce);
				ForkJoinPool.managedBlock(fp );
				byte[] bytes = fp.getPage();
				if(bytes!=null) {
					String code = toHexString(getSHA(bytes));
					hashPageMap.put(url, code);
					return code;
				}
			} catch (InterruptedException | NoSuchAlgorithmException e) {
				return null;
			}
			return null;
		}
		
	}
	
	public static void main(String[] args) {
		ExecutorService executorSerivce = Executors.newFixedThreadPool(10);
		ForkJoinPool forkJoinPool = ForkJoinPool.commonPool();
		
		MyRecursiveTask task1 = new MyRecursiveTask("https://www.yahoo.com", executorSerivce);
		MyRecursiveTask task2 = new MyRecursiveTask("https://www.google.com", executorSerivce);
		
		Future<String> f1 = forkJoinPool.submit(task1);
		Future<String> f2 = forkJoinPool.submit(task2);	
		try {
			String res1 = f1.get();
			String res2 = f2.get();
			System.out.println(res1);
			System.out.println(res2);
			executorSerivce.shutdown();
		} catch (InterruptedException | ExecutionException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
    public static byte[] getSHA(byte[] input) throws NoSuchAlgorithmException
    { 
        // Static getInstance method is called with hashing SHA 
        MessageDigest md = MessageDigest.getInstance("SHA-256"); 
  
        // digest() method called 
        // to calculate message digest of an input 
        // and return array of byte
        return md.digest(input); 
    }
    
    public static String toHexString(byte[] hash)
    {
        // Convert byte array into signum representation 
        BigInteger number = new BigInteger(1, hash); 
  
        // Convert message digest into hex value 
        StringBuilder hexString = new StringBuilder(number.toString(16)); 
  
        // Pad with leading zeros
        while (hexString.length() < 32) 
        { 
            hexString.insert(0, '0'); 
        } 
  
        return hexString.toString(); 
    }
}

This article is also published at GeeksForGeeks

Apache HttpClient hangs

Recently we were trying to hit a website in multiple threads and get the response to improve the speed. But unfortunately the HttpClient hangs when we try to hit it from multiple threads. Following is my code which does not work.


import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

public class HttpClientTest {
	public static class CallableRequest implements Callable<CloseableHttpResponse> {
		CloseableHttpClient client;
		
		public CallableRequest(CloseableHttpClient client) {
			this.client = client;
		}
		
		@Override
		public CloseableHttpResponse call() throws Exception {
			HttpGet request = new HttpGet("https://www.yahoo.com");
			CloseableHttpResponse response = client.execute(request);
			return response;
		}
		
	}
	public static void main(String[] args) throws IOException, InterruptedException, ExecutionException {
		CloseableHttpClient client = HttpClients.createDefault();
		ExecutorService executorSerivce = Executors.newFixedThreadPool(10);
		List<Future<CloseableHttpResponse>> respList = new ArrayList<>();
		
		for(int i=0; i<10; i++) {
			CallableRequest req = new CallableRequest(client);
			Future<CloseableHttpResponse> future = executorSerivce.submit(req);
			respList.add(future);
		}

		for(Future<CloseableHttpResponse> respFuture: respList) {
			CloseableHttpResponse resp = respFuture.get();
			System.out.println("Status code:" + resp.getStatusLine().getStatusCode());
			resp.close();
		}
		executorSerivce.shutdown();
	}

}

The HttpClient looks like to have an internal queue and process a limited number of open requests. It can process new requests only after response of earlier requests is closed.

I am closing the response after reading it, so it should free up the open requests and it should work. Right? But it does not work.  

It is not working because I am submitting 10 requests and waiting for response in sequence. But the sequence of submitting the request and execution of them by HttpClient can be different. If HttpClient can process only 4 concurrent requests and your first request happen to be 5th request in HttpClient then you will be waiting for your first request to complete and it will never complete because some other 4 requests need to be closed for your first request to be processed. The 4 request which are completed are later in your list. That is why this program hanged.

The Solution

The solution is to wait for results in sequence of completion and not in sequence of submission. For doing that you need to use ExecutorCompletionService for submitting your request. It has a method take() which returns you next completed task. You can process the results of you requests in sequence in which they are getting completed. This way you HttpClient will not get locked up. Following is the code:


import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class HttpClientTest2 {
	public static class CallableRequest implements Callable<CloseableHttpResponse> {
		CloseableHttpClient client;
		
		public CallableRequest(CloseableHttpClient client) {
			this.client = client;
		}
		
		@Override
		public CloseableHttpResponse call() throws Exception {
			HttpGet request = new HttpGet("https://www.yahoo.com");
			CloseableHttpResponse response = client.execute(request);
			return response;
		}
		
	}
	public static void main(String[] args) throws IOException, InterruptedException, ExecutionException {
		CloseableHttpClient client = HttpClients.createDefault();
		ExecutorService executorSerivce = Executors.newFixedThreadPool(10);
		CompletionService<CloseableHttpResponse> completionService = new ExecutorCompletionService<>(executorSerivce);
		List<Future<CloseableHttpResponse>> respList = new ArrayList<>();
		
		for(int i=0; i<10; i++) {
			CallableRequest req = new CallableRequest(client);
			Future<CloseableHttpResponse> future = completionService.submit(req);
			respList.add(future);
		}

		for(int i=0; i<10; i++) {
			CloseableHttpResponse resp = completionService.take().get();
			System.out.println("Status code:" + resp.getStatusLine().getStatusCode());
			resp.close();
		}
		executorSerivce.shutdown();
	}

}

This code works perfectly with reordering of processing of the requests which can happen in multi-threading.

Saturday, 12 June 2021

Substituting environment variables in tomcat configuration file

I was looking to run my application running under tomcat in Kubernetes cluster. For that I need to create a container image. I don't want to put username and passwords of database in the image. But at the same time I need it in tomcat configuration. I was looking for a way to  substitute environment variables in tomcat configuration.

In Kubernetes we can populate environment variables from secrets managed by Kubernetes. This way you can keep your secrets and containerize your application also.

You need to add following argument to CATALINA_OPTS


-Dorg.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource

 

After passing this argument you can use environment variables same way as system properties. You can use it like this:


${ENV_VARIABE_NAME}


Substituting environment variables in Wildfly config

 I was looking a way to run my Wildfly application in Kubernetes. I realized that my database hostname, database name, username and passwords are written in Wildfly's standalone-full.xml file. I don't want to make these details a part of container image. This information is secret and confidential. Kubernetes provide a way to manage secrets and can make these secret available as environment variables.

It will be good if I can use these environment variables in my config file. This way I don't have to hard code sensitive  information in container image. Luckily Wildfly has a way of doing this. This is how you use it.


                <datasource jta="true" jndi-name="java:/PostgresDS" pool-name="PostgresDS" enabled="true" use-java-context="true">
                    <connection-url>jdbc:postgresql://${env.DBHOST}:5432/${env.DBNAME}</connection-url>
                    <driver>postgresql</driver>
                    <pool>
                        <min-pool-size>20</min-pool-size>
                        <initial-pool-size>20</initial-pool-size>
                        <max-pool-size>200</max-pool-size>
                        <prefill>true</prefill>
                    </pool>
                    <security>
                        <user-name>${env.DBUSER}</user-name>
                        <password>${env.DBPASSWORD}</password>
                    </security>
                </datasource>

As you can see environment variable name are prefixed with "env." it tells wildfly that we are looking for environment and not for system property. You can specify a default value also in case environment variable is undefined syntax for that is:


${env.DBUSER:sampleuser}

Here sampleuser is the default value.

Saturday, 12 September 2020

Faster way of creating ZIP file

Zip file is normally used for distributing files and directories.  Zip file serves two purposes one is compressing the file and another of creating a achive of multiple files and directories. Now a days with faster internet speed and precompressed files like jpg and mp4 etc. It will be faster to disable compression altogether to speed up the compression. I wrote a simple program in java to create zip file without compression. I was using a mp4 file as content of zip file. With compression enabled it was taking 5 time more time and giving a compression of 5% which is not worth for time spent. Following is the source code:


package zipperformance;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import java.util.zip.CRC32;

public class ZipCreater {
	public static void main(String[] args) {
		
		long startTime = System.currentTimeMillis();
		zipDirectory("files", "test.zip");
		long endTime = System.currentTimeMillis();
		
		System.out.println(endTime-startTime);
		
		
	}
	
    public static void zipDirectory( String dirPath, String zipFilePath )
    {

        FileOutputStream fos = null;
        try
        {
            // Create the file output streams for both the file and the zip.

            File file = new File( zipFilePath );
            if(file.getParentFile()!=null) {
            	file.getParentFile().mkdirs();
            }
            fos = new FileOutputStream( zipFilePath );
            BufferedOutputStream bos = new BufferedOutputStream(fos);
            ZipOutputStream zos = new ZipOutputStream( fos );
            zos.setLevel(Deflater.BEST_SPEED);
            dirFunc( dirPath, dirPath, zos );

            // Close the file output streams for both the file and the zip.
            zos.flush();
            zos.close();
            fos.close();
        }
        catch ( IOException e )
        {
        	e.printStackTrace();
        }
    }
    
    private static void dirFunc( String dirName, String baseExportPath, ZipOutputStream zos )
    {
        try
        {
            File dirObj = new File( dirName );
            if ( dirObj.exists() == true )
            {
                if ( dirObj.isDirectory() == true )
                {
                    // Create an array of File objects, one for each file or directory in dirObj.
                    File[] fileList = dirObj.listFiles();
                    // Loop through File array and display.
                    for ( int i = 0; i < fileList.length; i++ )
                    {
                        if ( fileList[i].isDirectory() )
                        {
                            dirFunc( fileList[i].getPath(), baseExportPath, zos );
                        }
                        else if ( fileList[i].isFile() )

                        {
                            // Call the zipFunc function
                            zipFunc( fileList[i].getPath(), baseExportPath, zos );
                        }
                    }
                }
            }
        }
        catch ( Exception e )
        {
            e.printStackTrace();
        }
    }

    private static void zipFunc( String filePath, String baseExportPath, ZipOutputStream zos )
    {
        try
        {
            String absolutePath = filePath;
            String prefixFilePath = baseExportPath;

            int index = prefixFilePath.length();

            filePath = filePath.substring( ++index );

            // Create a file input stream and a buffered input stream.
            long size = Files.size(Paths.get(absolutePath));
            FileInputStream fis = new FileInputStream( absolutePath );
            BufferedInputStream bis = new BufferedInputStream( fis );
            CRC32 crc32 = new CRC32();
            byte[] data = new byte[1024*128];
            int byteCount;

            // Create a loop that reads from the buffered input stream and writes
            // to the zip output stream until the bis has been entirely read.
            while ( ( byteCount = bis.read( data, 0, data.length ) ) > -1 )
            {
                crc32.update( data, 0, byteCount );
            }
            long crc=crc32.getValue();

            // Create a Zip Entry and put it into the archive (no data yet).
            ZipEntry fileEntry = new ZipEntry( filePath );
            fileEntry.setMethod(ZipEntry.STORED);
            fileEntry.setSize(size);
            fileEntry.setCrc(crc);
            zos.putNextEntry( fileEntry );

            bis.close();
            fis = new FileInputStream( absolutePath );
            bis = new BufferedInputStream( fis );
            
            // Create a loop that reads from the buffered input stream and writes
            // to the zip output stream until the bis has been entirely read.
            while ( ( byteCount = bis.read( data, 0, data.length ) ) > -1 )
            {
                zos.write( data, 0, byteCount );
            }
        }
        catch ( IOException e )
        {
        	e.printStackTrace();
        }
    }
}


Monday, 20 July 2020

java.lang.OutOfMemoryError: Unable to create new native thread

Recently I received the error "java.lang.OutOfMemoryError: Unable to create new native thread" and when we debugged the issue on the linux machine the root cause was not related to memory but something totally different.  In Java when OS denies to create more threads because limit of number of processes have hit the limit then this error get mapped to java.lang.OutOfMemoryError because there is no specific error defined in java for capturing denial of creation of new thread due to hitting limit of number of processes.

If you want to check limits on a linux machine then you need to run the following command:

$ ulimit -a
core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 62837
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 16384
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 10240
cpu time               (seconds, -t) unlimited
max user processes              (-u) 1024
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited


"max user process" define maximum number of child processes/threads a root level unix process can open. There are soft limits and hard limits, soft limits can be set on a process and it applicable to that process and child processes and hard limit is applicable to all processes of that user.

In our case soft limit was configured to 1024 and we were trying to create more number of threads.
The limit is defined in file "/etc/security/limits.d/90-nproc.conf" we changed number of processes to 2048 and our application started working. 


# Default limit for number of user's processes to prevent
# accidental fork bombs.
# See rhbz #432903 for reasoning.

*          soft    nproc     1024
root       soft    nproc     unlimited

In Java denial of any resource by OS to application maps to java.lang.OutOfMemoryError and real reason might not be related to memory.

Setting Compression Level in GZIPOutputStream

Most of the time people want to compress files which they are generating to save space on disk and bandwidth in transmission. Apart from saving space compression can actually speed up the application because of low disk usage because of small file size. For this the compression and decompression need to done in memory and not after writing whole uncompressed contents to disk.

There are two formats which can be used if you are generating files from Java. One is GZIPOutputStream which is used for generating GZIP files, other is ZipOutputStream which is used for generating ZIP files. There is one basic difference between GZIP and ZIP file. GZIP file can contain only one file inside it and name of the file contained inside it is optional and while ZIP file is an archive of multiple files and name of the files contained in a ZIP file is mandatory while creating a ZIP file. Because of presence of multiple files inside a ZIP file. ZIP file cannot be passed to a filter which will decompress a ZIP file on the fly from an input stream because filter can't select one file out of multiple files which may be present in the ZIP file.

For seamless processing of compressed file while reading GZIP format is most suitable one. But unfortunately Java API for GZIPOutputStream lacks one method which can be used to controlling compression level to achieve BEST_SPEED or BEST_COMPRESSION as per your need. This facility is available in ZipOutputStream. Sometime people just use ZipOutput stream by setting compression level to BEST_SPEED to gain performance  when they actually need GZIPOutputStream for compressing their data. It create problem for the reader because now he need to handle a archive which can potentially contain multiple file rather than a compressed file. In memory filters can't be used for decompression because of possibility of multiple files in ZIP file. Therefore there are no libraries which can provide in memory filter for reading ZIP file contents as a stream of data.

Fortunately you can set compression level in GZIPOutputStream also by creating sub class of GZIPOutputStream and exposing setLevel(int level) method in your subclass.  We did it in our code and achieved even slightly better results than using ZipOutputStream with BEST_SPEED compression level. Following is comparison when compressing a 5.4 GB file:


Zip compression with BEST_SPEED        48078219 bytes     80 seconds
GZip compression with BEST_SPEED     48078113 bytes     78 seconds

Here is the code for MyGZIPOutputStream class:

import java.util.zip.*;
import java.io.*;

public class MyGZIPOutputStream extends GZIPOutputStream
{
    /**
     * Creates a new output stream with the specified buffer size.
     * @param out the output stream
     * @param size the output buffer size
     * @exception IOException If an I/O error has occurred.
     * @exception IllegalArgumentException if size is <= 0
     */
    public MyGZIPOutputStream(OutputStream out, int size) throws IOException {
        super(out, size);
    }

    /**
     * Creates a new output stream with a default buffer size.
     * @param out the output stream
     * @exception IOException If an I/O error has occurred.
     */
    public MyGZIPOutputStream(OutputStream out) throws IOException {
        this(out, 512);
    }

    /**
     * Sets the compression level for subsequent entries which are DEFLATED.
     * The default setting is DEFAULT_COMPRESSION.
     * @param level the compression level (0-9)
     * @exception IllegalArgumentException if the compression level is invalid
     */
    public void setLevel(int level) {
        def.setLevel(level);
    }
}

Sample file for using it BEST_SPEED compression in GZIPOutputStream:

import java.io.BufferedWriter;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.util.zip.*;

public class GZipCompression {

    public static void main(String[] args) throws IOException {
        compressInputFile("a.txt", "a.txt.gz");
    }

    public static void compressInputFile(String inputFileName,
            String outputFileName) throws IOException {
        FileOutputStream fos = new FileOutputStream(new File(outputFileName));
        MyGZIPOutputStream gzos = null;
        byte[] buffer = new byte[1024];
        gzos = new MyGZIPOutputStream(fos);
        gzos.setLevel(Deflater.BEST_SPEED);
        long startTime = System.currentTimeMillis();              

        FileInputStream fis = new FileInputStream(inputFileName);

        int length;
        while ((length = fis.read(buffer)) > 0) {
            gzos.write(buffer, 0, length);
        }
        fis.close();
        gzos.close();

        long endTime = System.currentTimeMillis();
        System.out.println("Time taken to gzip "+ (endTime-startTime) + " miliseconds.");
    }
}