I'm trying to run autossh (on a VM running CentOS6), but it's exiting immediately with the help message. I think this is a system issue because when I run it with the exact same parameters on another computer (running Ubuntu 14.04) it works fine. It's also fine when I run the same command but with ssh instead of autossh. So I tried strace to see if anything's wrong there. But if there is I'm not sure what it is. Any ideas?
Here's the autossh command: autossh -oStrictHostKeyChecking=no -oServerAliveInterval=15 -oServerAliveCountMax=4 -L 3130:localhost:3130 -N -i /path/to/some.pem user#remotehost
Here's the strace output (note myserver is an entry in .ssh/config that contains the same parameters as the previous command):
execve("/usr/local/bin/autossh", ["autossh", "myserver"], [/* 55 vars */]) = 0
brk(0) = 0xefc000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f26193cc000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=36751, ...}) = 0
mmap(NULL, 36751, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f26193c3000
close(3) = 0
open("/lib64/libnsl.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p#\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=113432, ...}) = 0
mmap(NULL, 2198192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2618f95000
mprotect(0x7f2618fab000, 2093056, PROT_NONE) = 0
mmap(0x7f26191aa000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f26191aa000
mmap(0x7f26191ac000, 6832, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f26191ac000
close(3) = 0
open("/lib64/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\356\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1920936, ...}) = 0
mmap(NULL, 3750152, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2618c01000
mprotect(0x7f2618d8b000, 2097152, PROT_NONE) = 0
mmap(0x7f2618f8b000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18a000) = 0x7f2618f8b000
mmap(0x7f2618f90000, 18696, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2618f90000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f26193c2000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f26193c1000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f26193c0000
arch_prctl(ARCH_SET_FS, 0x7f26193c1700) = 0
mprotect(0x7f2618f8b000, 16384, PROT_READ) = 0
mprotect(0x7f26191aa000, 4096, PROT_READ) = 0
mprotect(0x7f26193cd000, 4096, PROT_READ) = 0
munmap(0x7f26193c3000, 36751) = 0
write(2, "usage: autossh [-V] [-M monitor_"..., 69usage: autossh [-V] [-M monitor_port[:echo_port]] [-f] [SSH_OPTIONS]
) = 69
write(2, "\n", 1
) = 1
write(2, " -M specifies monitor port. M"..., 238 -M specifies monitor port. May be overridden by environment
variable AUTOSSH_PORT. 0 turns monitoring loop off.
Alternatively, a port for an echo service on the remote
machine may be specified. (Normally port 7.)
) = 238
write(2, " -f run in background (autoss"..., 85 -f run in background (autossh handles this, and does not
pass it to ssh.)
) = 85
write(2, " -V print autossh version and"..., 39 -V print autossh version and exit.
) = 39
write(2, "\n", 1
) = 1
write(2, "Environment variables are:\n", 27Environment variables are:
) = 27
write(2, " AUTOSSH_GATETIME - how lo"..., 259 AUTOSSH_GATETIME - how long must an ssh session be established
before we decide it really was established
(in seconds). Default is 30 seconds; use of -f
flag sets this to 0.
) = 259
write(2, " AUTOSSH_LOGFILE - file t"..., 107 AUTOSSH_LOGFILE - file to log to (default is to use the syslog
facility)
) = 107
write(2, " AUTOSSH_LOGLEVEL - level "..., 49 AUTOSSH_LOGLEVEL - level of log verbosity
) = 49
write(2, " AUTOSSH_MAXLIFETIME - set th"..., 65 AUTOSSH_MAXLIFETIME - set the maximum time to live (seconds)
) = 65
write(2, " AUTOSSH_MAXSTART - max ti"..., 69 AUTOSSH_MAXSTART - max times to restart (default is no limit)
) = 69
write(2, " AUTOSSH_MESSAGE - messag"..., 74 AUTOSSH_MESSAGE - message to append to echo string (max 64 bytes)
) = 74
write(2, " AUTOSSH_PATH - path t"..., 53 AUTOSSH_PATH - path to ssh if not default
) = 53
write(2, " AUTOSSH_PIDFILE - write "..., 49 AUTOSSH_PIDFILE - write pid to this file
) = 49
write(2, " AUTOSSH_POLL - how of"..., 70 AUTOSSH_POLL - how often to check the connection (seconds)
) = 70
write(2, " AUTOSSH_FIRST_POLL - time b"..., 71 AUTOSSH_FIRST_POLL - time before first connection check (seconds)
) = 71
write(2, " AUTOSSH_PORT - port t"..., 61 AUTOSSH_PORT - port to use for monitor connection
) = 61
write(2, " AUTOSSH_DEBUG - turn l"..., 104 AUTOSSH_DEBUG - turn logging to maximum verbosity and log to
stderr
) = 104
write(2, "\n", 1
) = 1
exit_group(1) = ?
+++ exited with 1 +++
I had exactly the same problem with autossh 1.4e on CentOS Linux 7. autossh stopped immediately and printed the help, without even trying to connect to SSH.
The solution was to specify -M 0 on the command line:
autossh -M 0 \
-oStrictHostKeyChecking=no \
-oServerAliveInterval=15 \
-oServerAliveCountMax=4 \
-L 3130:localhost:3130 \
-N -i /path/to/some.pem user#remotehost
Come on ... the autossh exits with
write(2, "usage: autossh [-V] [-M monitor_"..., 69usage: autossh [-V] [-M monitor_port[:echo_port]] [-f] [SSH_OPTIONS]
which means that you specified wrong arguments and it is trying to explain you how the syntax looks like. To see what is wrong, there are several things you can try:
Use -vvv switches to ssh to see more verbose log from the ssh.
Use AUTOSSH_DEBUG environment variable to get some debug logs from autossh.
we've been configuring monitoring of our openshift origin cluster and noticed that when we oc get pods --all-namespaces that our router pods have a high (> 200) restart count over the past month.
NAMESPACE NAME READY STATUS RESTARTS AGE
default router-2-kactc 1/1 Running 231 23d
default router-2-liyxb 1/1 Running 212 23d
edit: I was able to track down logs just prior to the pod crashing
fatal error: concurrent map read and map write
goroutine 44 [running]:
runtime.throw(0x38e04e0, 0x21)
/usr/local/go/src/runtime/panic.go:530 +0x90 fp=0xc820991238 sp=0xc820991220
runtime.mapaccess2(0x2820680, 0xc820650480, 0xc8202995c0, 0x317ca60, 0xc82078c750)
/usr/local/go/src/runtime/hashmap.go:343 +0x5a fp=0xc820991280 sp=0xc820991238
reflect.mapaccess(0x2820680, 0xc820650480, 0xc8202995c0, 0xc820650480)
/usr/local/go/src/runtime/hashmap.go:993 +0x35 fp=0xc8209912b0 sp=0xc820991280
reflect.Value.MapIndex(0x2820680, 0xc820206240, 0x95, 0x2828680, 0xc8202995c0, 0x98, 0x0, 0x0, 0x0)
/usr/local/go/src/reflect/value.go:1041 +0x14a fp=0xc820991338 sp=0xc8209912b0
text/template.(*state).walkRange(0xc8209918c0, 0x329f660, 0xc820206230, 0x99, 0xc820143e80)
/usr/local/go/src/text/template/exec.go:327 +0x885 fp=0xc820991538 sp=0xc820991338
text/template.(*state).walk(0xc8209918c0, 0x329f660, 0xc820206230, 0x99, 0x7f161f2bd1b0, 0xc820143e80)
/usr/local/go/src/text/template/exec.go:226 +0x264 fp=0xc820991680 sp=0xc820991538
text/template.(*state).walk(0xc8209918c0, 0x329f660, 0xc820206230, 0x99, 0x7f161f300540, 0xc8206484e0)
/usr/local/go/src/text/template/exec.go:223 +0x6a1 fp=0xc8209917c8 sp=0xc820991680
text/template.(*Template).Execute(0xc82056ad40, 0x7f161f2b8370, 0xc820074378, 0x329f660, 0xc820206230, 0x0, 0x0)
/usr/local/go/src/text/template/exec.go:178 +0x2a7 fp=0xc820991908 sp=0xc8209917c8
github.com/openshift/origin/pkg/router/template.(*templateRouter).writeConfig(0xc82064a340, 0x0, 0x0)
/go/src/github.com/openshift/origin/pkg/router/template/router.go:283 +0x892 fp=0xc820991d48 sp=0xc820991908
github.com/openshift/origin/pkg/router/template.(*templateRouter).commitAndReload(0xc82064a340, 0x0, 0x0)
/go/src/github.com/openshift/origin/pkg/router/template/router.go:231 +0x14d fp=0xc820991db0 sp=0xc820991d48
github.com/openshift/origin/pkg/router/template.(*templateRouter).(github.com/openshift/origin/pkg/router/template.commitAndReload)-fm(0x0, 0x0)
/go/src/github.com/openshift/origin/pkg/router/template/router.go:157 +0x2e fp=0xc820991dd0 sp=0xc820991db0
github.com/openshift/origin/pkg/util/ratelimiter.(*RateLimitedFunction).handleOne(0xc820650390, 0x33272e0, 0xc820650390)
/go/src/github.com/openshift/origin/pkg/util/ratelimiter/ratelimiter.go:49 +0x48 fp=0xc820991e08 sp=0xc820991dd0
github.com/openshift/origin/pkg/util/ratelimiter.(*RateLimitedFunction).RunUntil.func1()
/go/src/github.com/openshift/origin/pkg/util/ratelimiter/ratelimiter.go:42 +0x6a fp=0xc820991e50 sp=0xc820991e08
k8s.io/kubernetes/pkg/util/wait.JitterUntil.func1(0xc8204cfc10)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:66 +0x4b fp=0xc820991e80 sp=0xc820991e50
k8s.io/kubernetes/pkg/util/wait.JitterUntil(0xc8204cfc10, 0x0, 0x0, 0xc82005a540)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:67 +0x68 fp=0xc820991f40 sp=0xc820991e80
k8s.io/kubernetes/pkg/util/wait.Until(0xc8204cfc10, 0x0, 0xc82005a540)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:47 +0x3e fp=0xc820991f68 sp=0xc820991f40
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc820991f70 sp=0xc820991f68
created by github.com/openshift/origin/pkg/util/ratelimiter.(*RateLimitedFunction).RunUntil
/go/src/github.com/openshift/origin/pkg/util/ratelimiter/ratelimiter.go:42 +0x7d
goroutine 1 [select (no cases), 129 minutes]:
github.com/openshift/origin/pkg/cmd/infra/router.(*TemplateRouterOptions).Run(0xc820079c80, 0x0, 0x0)
/go/src/github.com/openshift/origin/pkg/cmd/infra/router/template.go:203 +0x85e
github.com/openshift/origin/pkg/cmd/infra/router.NewCommandTemplateRouter.func1(0xc8200f8600, 0x5b9a280, 0x0, 0x0)
/go/src/github.com/openshift/origin/pkg/cmd/infra/router/template.go:111 +0x1ad
github.com/spf13/cobra.(*Command).execute(0xc8200f8600, 0xc82005e140, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/github.com/spf13/cobra/command.go:572 +0x85a
github.com/spf13/cobra.(*Command).ExecuteC(0xc8200f8600, 0xc8200f8600, 0x0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/github.com/spf13/cobra/command.go:662 +0x53f
github.com/spf13/cobra.(*Command).Execute(0xc8200f8600, 0x0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/github.com/spf13/cobra/command.go:618 +0x2d
main.main()
/go/src/github.com/openshift/origin/cmd/openshift/openshift.go:27 +0x180
goroutine 17 [syscall, 129 minutes, locked to thread]:
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1998 +0x1
goroutine 19 [chan receive]:
github.com/golang/glog.(*loggingT).flushDaemon(0x5b72140)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/github.com/golang/glog/glog.go:879 +0x67
created by github.com/golang/glog.init.1
/go/src/github.com/openshift/origin/Godeps/_workspace/src/github.com/golang/glog/glog.go:410 +0x297
goroutine 9 [syscall, 9 minutes]:
os/signal.signal_recv(0x7f161f2b4050)
/usr/local/go/src/runtime/sigqueue.go:116 +0x132
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
/usr/local/go/src/os/signal/signal_unix.go:28 +0x37
goroutine 45 [select]:
k8s.io/kubernetes/pkg/client/cache.(*Reflector).watchHandler(0xc82041a0a0, 0x7f161f2d1d30, 0xc8208aabd0, 0xc820665bf0, 0xc8203f0060, 0xc82005ae40, 0x0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:320 +0x13f7
k8s.io/kubernetes/pkg/client/cache.(*Reflector).ListAndWatch(0xc82041a0a0, 0xc82005ae40, 0x0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:287 +0x125f
k8s.io/kubernetes/pkg/client/cache.(*Reflector).Run.func1()
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:160 +0x2c
k8s.io/kubernetes/pkg/util/wait.JitterUntil.func1(0xc8203f5200)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:66 +0x4b
k8s.io/kubernetes/pkg/util/wait.JitterUntil(0xc8203f5200, 0x3b9aca00, 0x0, 0xc82005ae40)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:67 +0x68
k8s.io/kubernetes/pkg/util/wait.Until(0xc8203f5200, 0x3b9aca00, 0xc82005ae40)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:47 +0x3e
created by k8s.io/kubernetes/pkg/client/cache.(*Reflector).Run
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:160 +0x84
goroutine 46 [select]:
k8s.io/kubernetes/pkg/client/cache.(*Reflector).watchHandler(0xc82041a1e0, 0x7f161f2d1d30, 0xc8208aa450, 0xc8205cdc00, 0xc8208c2000, 0xc82005ae40, 0x0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:320 +0x13f7
k8s.io/kubernetes/pkg/client/cache.(*Reflector).ListAndWatch(0xc82041a1e0, 0xc82005ae40, 0x0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:287 +0x125f
k8s.io/kubernetes/pkg/client/cache.(*Reflector).Run.func1()
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:160 +0x2c
k8s.io/kubernetes/pkg/util/wait.JitterUntil.func1(0xc8203f5260)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:66 +0x4b
k8s.io/kubernetes/pkg/util/wait.JitterUntil(0xc8203f5260, 0x3b9aca00, 0x0, 0xc82005ae40)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:67 +0x68
k8s.io/kubernetes/pkg/util/wait.Until(0xc8203f5260, 0x3b9aca00, 0xc82005ae40)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:47 +0x3e
created by k8s.io/kubernetes/pkg/client/cache.(*Reflector).Run
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:160 +0x84
goroutine 47 [runnable]:
k8s.io/kubernetes/pkg/util/wait.JitterUntil(0xc8203f52e0, 0x0, 0x0, 0xc82005ae40)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:74 +0x13c
k8s.io/kubernetes/pkg/util/wait.Until(0xc8203f52e0, 0x0, 0xc82005ae40)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:47 +0x3e
k8s.io/kubernetes/pkg/util/wait.Forever(0xc8203f52e0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:41 +0x37
created by github.com/openshift/origin/pkg/router/controller.(*RouterController).Run
/go/src/github.com/openshift/origin/pkg/router/controller/controller.go:46 +0x1da
goroutine 48 [semacquire, 9 minutes]:
sync.runtime_Syncsemacquire(0xc8203e2418)
/usr/local/go/src/runtime/sema.go:241 +0x201
sync.(*Cond).Wait(0xc8203e2408)
/usr/local/go/src/sync/cond.go:63 +0x9b
github.com/openshift/origin/pkg/client/cache.(*EventQueue).Pop(0xc8203e23f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/openshift/origin/pkg/client/cache/eventqueue.go:273 +0xd9
github.com/openshift/origin/pkg/router/controller/factory.(*RouterControllerFactory).Create.func1(0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/openshift/origin/pkg/router/controller/factory/factory.go:71 +0x42
github.com/openshift/origin/pkg/router/controller.(*RouterController).HandleEndpoints(0xc82064d540)
/go/src/github.com/openshift/origin/pkg/router/controller/controller.go:91 +0x30
github.com/openshift/origin/pkg/router/controller.(*RouterController).HandleEndpoints-fm()
/go/src/github.com/openshift/origin/pkg/router/controller/controller.go:47 +0x20
k8s.io/kubernetes/pkg/util/wait.JitterUntil.func1(0xc8203f52f0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:66 +0x4b
k8s.io/kubernetes/pkg/util/wait.JitterUntil(0xc8203f52f0, 0x0, 0x0, 0xc82005ae40)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:67 +0x68
k8s.io/kubernetes/pkg/util/wait.Until(0xc8203f52f0, 0x0, 0xc82005ae40)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:47 +0x3e
k8s.io/kubernetes/pkg/util/wait.Forever(0xc8203f52f0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/util/wait/wait.go:41 +0x37
created by github.com/openshift/origin/pkg/router/controller.(*RouterController).Run
/go/src/github.com/openshift/origin/pkg/router/controller/controller.go:47 +0x239
goroutine 49 [chan receive, 9 minutes]:
github.com/openshift/origin/pkg/util/proc.StartReaper.func1()
/go/src/github.com/openshift/origin/pkg/util/proc/reaper.go:23 +0x193
created by github.com/openshift/origin/pkg/util/proc.StartReaper
/go/src/github.com/openshift/origin/pkg/util/proc/reaper.go:35 +0x79
goroutine 82 [select, 129 minutes, locked to thread]:
runtime.gopark(0x3e67cf8, 0xc820023f28, 0x3573d80, 0x6, 0x430218, 0x2)
/usr/local/go/src/runtime/proc.go:262 +0x163
runtime.selectgoImpl(0xc820023f28, 0x0, 0x18)
/usr/local/go/src/runtime/select.go:392 +0xa67
runtime.selectgo(0xc820023f28)
/usr/local/go/src/runtime/select.go:215 +0x12
runtime.ensureSigM.func1()
/usr/local/go/src/runtime/signal1_unix.go:279 +0x358
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1998 +0x1
goroutine 2218 [IO wait]:
net.runtime_pollWait(0x7f161f2cd258, 0x72, 0xc82046e800)
/usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc8204c3f00, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8204c3f00, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8204c3ea0, 0xc82046e800, 0x400, 0x400, 0x0, 0x7f161f2b4028, 0xc82005e080)
/usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc820024288, 0xc82046e800, 0x400, 0x400, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:172 +0xe4
crypto/tls.(*block).readFromUntil(0xc8208aa360, 0x7f161f3005f8, 0xc820024288, 0x5, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:460 +0xcc
crypto/tls.(*Conn).readRecord(0xc820676300, 0x3e67e17, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:562 +0x2d1
crypto/tls.(*Conn).Read(0xc820676300, 0xc820840000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:939 +0x167
net/http.noteEOFReader.Read(0x7f161f3006b0, 0xc820676300, 0xc8207b1248, 0xc820840000, 0x1000, 0x1000, 0xc820027800, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1683 +0x67
net/http.(*noteEOFReader).Read(0xc8209f0380, 0xc820840000, 0x1000, 0x1000, 0xfc19b4, 0x0, 0x0)
<autogenerated>:284 +0xd0
bufio.(*Reader).fill(0xc820979e60)
/usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).ReadSlice(0xc820979e60, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:328 +0x21a
net/http/internal.readChunkLine(0xc820979e60, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/internal/chunked.go:110 +0x53
net/http/internal.(*chunkedReader).beginChunk(0xc8208aa3f0)
/usr/local/go/src/net/http/internal/chunked.go:47 +0x39
net/http/internal.(*chunkedReader).Read(0xc8208aa3f0, 0xc820151600, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/internal/chunked.go:77 +0xb7
net/http.(*body).readLocked(0xc820860200, 0xc820151600, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:651 +0xa1
net/http.(*body).Read(0xc820860200, 0xc820151600, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:643 +0x110
net/http.(*bodyEOFSignal).Read(0xc820860280, 0xc820151600, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1609 +0x26a
encoding/json.(*Decoder).refill(0xc8203924e0, 0x0, 0x0)
/usr/local/go/src/encoding/json/stream.go:152 +0x287
encoding/json.(*Decoder).readValue(0xc8203924e0, 0x1, 0x0, 0x0)
/usr/local/go/src/encoding/json/stream.go:128 +0x413
encoding/json.(*Decoder).Decode(0xc8203924e0, 0x275c5c0, 0xc8208602c0, 0x0, 0x0)
/usr/local/go/src/encoding/json/stream.go:57 +0x159
k8s.io/kubernetes/pkg/watch/json.(*Decoder).Decode(0xc8208aa420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/watch/json/decoder.go:50 +0xa5
k8s.io/kubernetes/pkg/watch.(*StreamWatcher).receive(0xc8208aa450)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/watch/iowatcher.go:93 +0x104
created by k8s.io/kubernetes/pkg/watch.NewStreamWatcher
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/watch/iowatcher.go:60 +0xbf
goroutine 2243 [select]:
net/http.(*persistConn).readLoop(0xc820815ba0)
/usr/local/go/src/net/http/transport.go:1178 +0xd52
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:853 +0x10a6
goroutine 2244 [select]:
net/http.(*persistConn).writeLoop(0xc820815ba0)
/usr/local/go/src/net/http/transport.go:1273 +0x472
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:854 +0x10cb
goroutine 2220 [IO wait]:
net.runtime_pollWait(0x7f161f2cd198, 0x72, 0xc8209f6000)
/usr/local/go/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc82054c4c0, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc82054c4c0, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc82054c460, 0xc8209f6000, 0x8000, 0x8000, 0x0, 0x7f161f2b4028, 0xc82005e080)
/usr/local/go/src/net/fd_unix.go:250 +0x23a
net.(*conn).Read(0xc8200242d8, 0xc8209f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:172 +0xe4
crypto/tls.(*block).readFromUntil(0xc820a35b00, 0x7f161f3005f8, 0xc8200242d8, 0x5, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:460 +0xcc
crypto/tls.(*Conn).readRecord(0xc820676600, 0x3e67e17, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:562 +0x2d1
crypto/tls.(*Conn).Read(0xc820676600, 0xc82072a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:939 +0x167
net/http.noteEOFReader.Read(0x7f161f3006b0, 0xc820676600, 0xc820815c08, 0xc82072a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1683 +0x67
net/http.(*noteEOFReader).Read(0xc82059b9c0, 0xc82072a000, 0x1000, 0x1000, 0xc8208aa480, 0x0, 0x0)
<autogenerated>:284 +0xd0
bufio.(*Reader).fill(0xc8207fc3c0)
/usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).ReadSlice(0xc8207fc3c0, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:328 +0x21a
net/http/internal.readChunkLine(0xc8207fc3c0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/internal/chunked.go:110 +0x53
net/http/internal.(*chunkedReader).beginChunk(0xc8208aab70)
/usr/local/go/src/net/http/internal/chunked.go:47 +0x39
net/http/internal.(*chunkedReader).Read(0xc8208aab70, 0xc820151c00, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/internal/chunked.go:77 +0xb7
net/http.(*body).readLocked(0xc8208604c0, 0xc820151c00, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:651 +0xa1
net/http.(*body).Read(0xc8208604c0, 0xc820151c00, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:643 +0x110
net/http.(*bodyEOFSignal).Read(0xc820860500, 0xc820151c00, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1609 +0x26a
encoding/json.(*Decoder).refill(0xc820392680, 0x0, 0x0)
/usr/local/go/src/encoding/json/stream.go:152 +0x287
encoding/json.(*Decoder).readValue(0xc820392680, 0x1, 0x0, 0x0)
/usr/local/go/src/encoding/json/stream.go:128 +0x413
encoding/json.(*Decoder).Decode(0xc820392680, 0x275c5c0, 0xc820860540, 0x0, 0x0)
/usr/local/go/src/encoding/json/stream.go:57 +0x159
k8s.io/kubernetes/pkg/watch/json.(*Decoder).Decode(0xc8208aaba0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/watch/json/decoder.go:50 +0xa5
k8s.io/kubernetes/pkg/watch.(*StreamWatcher).receive(0xc8208aabd0)
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/watch/iowatcher.go:93 +0x104
created by k8s.io/kubernetes/pkg/watch.NewStreamWatcher
/go/src/github.com/openshift/origin/Godeps/_workspace/src/k8s.io/kubernetes/pkg/watch/iowatcher.go:60 +0xbf
goroutine 2238 [select]:
net/http.(*persistConn).writeLoop(0xc8207b11e0)
/usr/local/go/src/net/http/transport.go:1273 +0x472
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:854 +0x10cb
goroutine 2237 [select]:
net/http.(*persistConn).readLoop(0xc8207b11e0)
/usr/local/go/src/net/http/transport.go:1178 +0xd52
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:853 +0x10a6
Is this normal? Should we be concerned?
For anyone else getting this issue, I was able to track down this github issue where this is discussed (and a PR later submitted to fix): https://github.com/openshift/origin/issues/8825
I have a small problem with AXTLS. I'm trying to establish a SSL communication with a server, but it looks like it double-frees something in "ssl_client_new" function.
The code creates a new ctx (ssl_ctx_new) and loads the certificates (ssl_obj_load) during initialization (no problem here), the socket is connected and when I call "ssl_client_new", libc detects a double free:
"double free or corruption (fasttop): 0x0809f740"
I started the application under gdb, and this is the backtrace:
Program received signal SIGABRT, Aborted. 0xb7fdd424 in
__kernel_vsyscall () (gdb) bt
#0 0xb7fdd424 in __kernel_vsyscall ()
#1 0xb7d371ef in __GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#2 0xb7d3a835 in __GI_abort () at abort.c:91
#3 0xb7d722fa in __libc_message (do_abort=2, fmt=0xb7e6a3bc "*** glibc detected *** %s: %s: 0x%s ***\n") at ../sysdeps/unix/sysv/linux/libc_fatal.c:201
#4 0xb7d7ce42 in malloc_printerr (action=<optimized out>, str=<optimized out>, ptr=0x809f740) at malloc.c:5007
#5 0x080712fd in asn1_name ()
#6 0x0806849f in x509_new ()
#7 0x080642cc in process_certificate ()
#8 0x08067caa in do_clnt_handshake ()
#9 0x08066675 in basic_read ()
#10 0x08067a07 in ssl_client_new ()
The ctx pointer and the sockets are ok, and last two parameters are NULL and 0. (http://axtls.sourceforge.net/dox/group_c_api.html#ga4eef242a559b06d72b862c1e5ab3d0a2)
Here is the strace for my socket:
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 8
fcntl64(8, F_GETFL) = 0x2 (flags O_RDWR)
fcntl64(8, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(8, {sa_family=AF_INET, sin_port=htons(443), sin_addr=inet_addr("81.12.132.173")}, 16) = -1 EINPROGRESS (Operation now in progress)
select(9, NULL, [8], NULL, {10, 0}) = 1 (out [8], left {9, 996818})
getsockopt(8, SOL_SOCKET, SO_ERROR, [0], [4]) = 0
fcntl64(8, F_SETFL, O_RDWR) = 0
fcntl64(8, F_GETFL) = 0x2 (flags O_RDWR)
fcntl64(8, F_SETFL, O_RDWR) = 0
write(8, "\26\3\1\0003\1\0\0/\3\1P#hf?\222Y\3046n\215\364\317\34-D8\311\270=\225"..., 56) = 56
read(8, "\26\3\1\0J", 5) = 5
read(8, "\2\0\0F\3\1Q\301_\210\267\200\352*}H\330\265\n;\33\253\31\24\320\377+\3\371\276g\362"..., 74) = 74
read(8, "\26\3\1\20)", 5) = 5
read(8, "\v\0\20%\0\20\"\0\5\0070\202\5\0030\202\3\353\240\3\2\1\2\2\23w\0\0~\263\3446"..., 4137) = 2632
read(8, "1\0270\25\6\3U\4\n\f\16Vodafone Group1)0'\6\3U"..., 1505) = 1505
So ssl handshake started.
And immediatly after this:
open("/dev/tty", O_RDWR|O_NOCTTY|O_NONBLOCK) = 9
writev(9, [{"*** glibc detected *** ", 23}, {"./HTTPS_TOOL", 11}, {": ", 2}, {"double free or corruption (fastt"..., 35}, {": 0x", 4}, {"082ba740", 8}, {" ***\n", 5}], 7*** glibc detected *** ./HTTPS_TOOL: double free or corruption (fasttop): 0x082ba740 ***
) = 88
Any ideas? Which can be the problem and what can I do?
Thank you.
Are you still getting the issue? I haven't had any reports of this and it may be a way the library is being used.
The library has been checked many times in Valgrind and there aren't any recent issues. Could you perhaps run it in Valgrind to verify?