...
| Code Block |
|---|
global
log 127.0.0.1 local2
chroot /var/lib/haproxy
stats socket /var/lib/haproxy/stats mode 660 level admin
user haproxy
group haproxy
daemon
ca-base /etc/pki/tls/certs
crt-base /etc/pki/tls/private
ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
ssl-default-bind-options no-sslv3
maxconn 2048
tune.ssl.default-dh-param 2048
defaults
log global
mode http
option forwardfor
# Do not use "option http-server-close", it causes S3 PUT incompatibility with some clients including FileFly!
option httplog
option dontlognull
timeout connect 5000
timeout client 50000
# This timeout should always be larger than gateway.cfg's [storage_cluster] indexerSocketTimeout
# and large enough to not timeout an S3 CopyObject or an SCSP COPY in a versioned bucket.
timeout server 130000
frontend www-http
bind 0.0.0.0:80
http-request set-header X-Forwarded-Proto http
http-request set-header X-Forwarded-Port 80
default_backend www-backend-scsp
acl iss3 hdr_sub(Authorization) AWS
acl iss3 url_reg [?&](AWSAccessKeyId|X-Amz-Credential)=
use_backend www-backend-s3 if iss3
frontend www-https
bind 0.0.0.0:443 ssl crt /etc/pki/tls/certs/YOUR_DOMAIN.pem
http-request set-header X-Forwarded-Proto https
http-request set-header X-Forwarded-Port 443
default_backend www-backend-scsp
acl iss3 hdr_sub(Authorization) AWS
acl iss3 url_reg [?&](AWSAccessKeyId|X-Amz-Credential)=
use_backend www-backend-s3 if iss3
frontend www-https-svc
bind 0.0.0.0:91 ssl crt /etc/pki/tls/certs/YOUR_DOMAIN.pem
http-request set-header X-Forwarded-Proto https
http-request set-header X-Forwarded-Port 91
default_backend www-backend-svc
backend www-backend-scsp
#redirectbalance schemeleastconn
https if !{ ssl_fc }# On HAPROXY <--- Uncomment this line if you want to force HTTPS
server gw1 YOUR_GATEWAY1_IP:8080 check
server gw2 YOUR_GATEWAY2_IP:8080 check
backend www-backend-s3 2.2
option httpchk HEAD /_admin/manage/version HTTP/1.1\r\nHost:\ haproxy-healthcheck
http-check expect status 200
# On HAPROXY > 2.2
#redirect#option schemehttpchk
https if !{ ssl_fc } #http-check connect
<--- Uncomment this#http-check linesend ifmeth youHEAD want to force HTTPS
server gw1 YOUR_GATEWAY1_IP:8090 check
server gw2 YOUR_GATEWAY2_IP:8090 check
backend www-backend-svc
# This rule rewrites CORS header to add the port number used on frontend
http-request replace-value Access-Control-Allow-Origin (.*) \1:91
#redirect scheme https if !{ ssl_fc } <--- Uncomment this line if you want to force HTTPS
server gw1 YOUR_GATEWAY1_IP:8091 check
server gw2 YOUR_GATEWAY2_IP:8091 check |
Start HAProxy:
systemctl restart haproxy
| Note |
|---|
If when restarting HAProxy this error is thrown “Starting frontend www-https-svc: cannot bind socket [0.0.0.0:91]”, either disable SELinux or run this command: setsebool -P haproxy_connect_any=1uri /_admin/manage/version ver HTTP/1.1 hdr Host haproxy-healthcheck
#http-check expect status 200
#redirect scheme https if !{ ssl_fc } <--- Uncomment this line if you want to force HTTPS
server gw1 YOUR_GATEWAY1_IP:8080 check inter 10s fall 3 rise 2
server gw2 YOUR_GATEWAY2_IP:8080 check inter 10s fall 3 rise 2
backend www-backend-s3
balance leastconn
# On HAPROXY < 2.2
option httpchk HEAD / HTTP/1.1\r\nHost:\ haproxy-healthcheck
http-check expect status 403
# On HAPROXY > 2.2
#option httpchk
#http-check connect
#http-check send meth HEAD uri / ver HTTP/1.1 hdr Host haproxy-healthcheck
#http-check expect status 403
#redirect scheme https if !{ ssl_fc } <--- Uncomment this line if you want to force HTTPS
server gw1 YOUR_GATEWAY1_IP:8090 check inter 10s fall 3 rise 2
server gw2 YOUR_GATEWAY2_IP:8090 check inter 10s fall 3 rise 2
backend www-backend-svc
balance leastconn
# On HAPROXY < 2.2
option httpchk HEAD / HTTP/1.1\r\nHost:\ haproxy-healthcheck
http-check expect status 401
# On HAPROXY > 2.2
#option httpchk
#http-check connect
#http-check send meth HEAD uri / ver HTTP/1.1 hdr Host haproxy-healthcheck
#http-check expect status 401
# This rule rewrites CORS header to add the port number used on frontend
http-request replace-value Access-Control-Allow-Origin (.*) \1:91
#redirect scheme https if !{ ssl_fc } <--- Uncomment this line if you want to force HTTPS
server gw1 YOUR_GATEWAY1_IP:8091 check inter 10s fall 3 rise 2
server gw2 YOUR_GATEWAY2_IP:8091 check inter 10s fall 3 rise 2 |
Start HAProxy:
systemctl restart haproxy
| Note |
|---|
If when restarting HAProxy this error is thrown “Starting frontend www-https-svc: cannot bind socket [0.0.0.0:91]”, either disable SELinux or run this command:
|
Loadbalancing healthcheck
If you have 2 or more gateways, it is recommended to use the http-check directives to verify the service is live.
Each protocol requires different healthcheck configuration.
SCSP Protocol
HAPROXY < 2.2
| Code Block |
|---|
option httpchk HEAD /_admin/manage/version HTTP/1.1\r\nHost:\ haproxy-healthcheck
http-check expect status 200 |
HAPROXY > 2.2
| Code Block |
|---|
option httpchk
http-check connect
http-check send meth HEAD uri /_admin/manage/version ver HTTP/1.1 hdr Host haproxy-healthcheck
http-check expect status 200 |
S3 Protocol
HAPROXY < 2.2
| Code Block |
|---|
option httpchk HEAD / HTTP/1.1\r\nHost:\ haproxy-healthcheck
http-check expect status 403 |
HAPROXY > 2.2
| Code Block |
|---|
option httpchk
http-check connect
http-check send meth HEAD uri / ver HTTP/1.1 hdr Host haproxy-healthcheck
http-check expect status 403 |
Admin Protocol ( cluster_admin )
HAPROXY < 2.2
| Code Block |
|---|
option httpchk HEAD / HTTP/1.1\r\nHost:\ haproxy-healthcheck
http-check expect status 401 |
HAPROXY > 2.2
| Code Block |
|---|
option httpchk
http-check connect
http-check send meth HEAD uri / ver HTTP/1.1 hdr Host haproxy-healthcheck
http-check expect status 401 |
Backend section for all protocols
| Code Block |
|---|
server gw1 YOUR_GATEWAY1_IP:8090 check inter 10s fall 3 rise 2
server gw2 YOUR_GATEWAY2_IP:8090 check inter 10s fall 3 rise 2 |
Example Logging with SCSP health check
In the following example I ran a while loop, to read an object every second… and while doing it shutdown 1 of the 2 gateways waited a few seconds then started it back up.
| Code Block |
|---|
Mar 15 08:01:54 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45870 [15/Mar/2024:08:01:54.044] www-http swarm-scsp/gw1 0/0/0/20/29 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.0|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:01:55 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45876 [15/Mar/2024:08:01:55.095] www-http swarm-scsp/gw2 0/0/0/20/28 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:01:56 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45880 [15/Mar/2024:08:01:56.144] www-http swarm-scsp/gw1 0/0/0/16/22 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.0|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:01:57 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45890 [15/Mar/2024:08:01:57.187] www-http swarm-scsp/gw2 0/0/0/20/29 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
**** Here I stop gw1 service... which produces 503's until haproxy detects it
Mar 15 08:02:01 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45894 [15/Mar/2024:08:01:58.236] www-http swarm-scsp/gw1 0/0/-1/-1/3003 503 216 - - SC-- 1/1/0/0/3 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:02 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45908 [15/Mar/2024:08:02:02.260] www-http swarm-scsp/gw2 0/0/0/20/29 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:06 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45914 [15/Mar/2024:08:02:03.310] www-http swarm-scsp/gw1 0/0/-1/-1/3006 503 216 - - SC-- 1/1/0/0/3 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:07 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45928 [15/Mar/2024:08:02:07.338] www-http swarm-scsp/gw2 0/0/0/19/27 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:11 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45932 [15/Mar/2024:08:02:08.387] www-http swarm-scsp/gw1 0/0/-1/-1/3006 503 216 - - SC-- 1/1/0/0/3 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:12 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45946 [15/Mar/2024:08:02:12.415] www-http swarm-scsp/gw2 0/0/0/14/21 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:16 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45954 [15/Mar/2024:08:02:13.456] www-http swarm-scsp/gw1 0/0/-1/-1/3006 503 216 - - SC-- 1/1/0/0/3 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {|} "GET /test14/test2 HTTP/1.1"
**** HAproxy figures out gw1 is down
Mar 15 08:02:16 cert-prod-haproxy haproxy[1411077]: [WARNING] (1411077) : Server swarm-scsp/gw1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.
Mar 15 08:02:16 cert-prod-haproxy haproxy[1411077]: Server swarm-scsp/gw1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.
Mar 15 08:02:16 cert-prod-haproxy haproxy[1411077]: Server swarm-scsp/gw1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.
**** At this point all calls go to gw2 only
Mar 15 08:02:17 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45968 [15/Mar/2024:08:02:17.483] www-http swarm-scsp/gw2 0/0/0/19/28 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:18 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45974 [15/Mar/2024:08:02:18.532] www-http swarm-scsp/gw2 0/0/0/18/24 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:19 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45978 [15/Mar/2024:08:02:19.577] www-http swarm-scsp/gw2 0/0/0/17/24 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:20 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45984 [15/Mar/2024:08:02:20.623] www-http swarm-scsp/gw2 0/0/0/16/23 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:21 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:45988 [15/Mar/2024:08:02:21.667] www-http swarm-scsp/gw2 0/0/0/16/25 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
*** Here I start gw1 back up, and Haproxy detects it.
Mar 15 08:02:36 cert-prod-haproxy haproxy[1411077]: [WARNING] (1411077) : Server swarm-scsp/gw1 is UP, reason: Layer7 check passed, code: 200, check duration: 18ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue.
Mar 15 08:02:36 cert-prod-haproxy haproxy[1411077]: Server swarm-scsp/gw1 is UP, reason: Layer7 check passed, code: 200, check duration: 18ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue.
Mar 15 08:02:36 cert-prod-haproxy haproxy[1411077]: Server swarm-scsp/gw1 is UP, reason: Layer7 check passed, code: 200, check duration: 18ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue.
*** Here we are back using both gateways
Mar 15 08:02:37 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:46086 [15/Mar/2024:08:02:37.293] www-http swarm-scsp/gw2 0/0/0/13/21 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:39 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:46090 [15/Mar/2024:08:02:38.336] www-http swarm-scsp/gw1 0/0/0/894/899 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.0|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:40 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:46100 [15/Mar/2024:08:02:40.256] www-http swarm-scsp/gw2 0/0/0/14/21 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.a|} "GET /test14/test2 HTTP/1.1"
Mar 15 08:02:41 cert-prod-haproxy haproxy[1411077]: 172.30.4.246:46106 [15/Mar/2024:08:02:41.299] www-http swarm-scsp/gw1 0/0/0/18/23 200 1058993 - - ---- 1/1/0/0/0 0/0 {|swarm.certlab.datacore.com|curl/7.29.0|||Basic YWRtaW46ZGF0YWNvcmU=} {CAStor Cluster/16.0.0|} "GET /test14/test2 HTTP/1.1" |
Create a Self-Signed SSL Certificate
...