[ Jan 17 08:28:44 Enabled. ]
[ Jan 17 08:28:44 Rereading configuration. ]
[ Jan 17 08:28:47 Rereading configuration. ]
[ Jan 17 08:28:51 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_72ed4737-9cfc-4e16-8b65-a8465f543320/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_72ed4737-9cfc-4e16-8b65-a8465f543320/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:28:52.047904983Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:28:52.048947511Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:28:52.048958651Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:28:52.066527381Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:28:52.226008691Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.386854434Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:38903"}
{"msg":"region 764d6ddf-285c-4a2b-8560-9e075924e32d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.415684405Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.420285924Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"uri":"/crucible/0/regions","method":"POST","req_id":"ca649f76-c6a6-4317-976e-ac335183d705","remote_addr":"[fd00:1122:3344:101::c]:38903","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":12925,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.420308444Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_72ed4737-9cfc-4e16-8b65-a8465f543320/crucible/regions/764d6ddf-285c-4a2b-8560-9e075924e32d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.429885168Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_72ed4737-9cfc-4e16-8b65-a8465f543320/crucible/regions/764d6ddf-285c-4a2b-8560-9e075924e32d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.429917798Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"764d6ddf-285c-4a2b-8560-9e075924e32d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/764d6ddf-285c-4a2b-8560-9e075924e32d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.49511726Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"region":"764d6ddf-285c-4a2b-8560-9e075924e32d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"764d6ddf-285c-4a2b-8560-9e075924e32d\" \"--data\" \"/data/regions/764d6ddf-285c-4a2b-8560-9e075924e32d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.497048883Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"region":"764d6ddf-285c-4a2b-8560-9e075924e32d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.628163939Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"region":"764d6ddf-285c-4a2b-8560-9e075924e32d","component":"worker"}
{"msg":"region 764d6ddf-285c-4a2b-8560-9e075924e32d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.62821537Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.628318101Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-764d6ddf-285c-4a2b-8560-9e075924e32d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.62878718Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-764d6ddf-285c-4a2b-8560-9e075924e32d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.634791022Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.64640609Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-764d6ddf-285c-4a2b-8560-9e075924e32d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.659183728Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/764d6ddf-285c-4a2b-8560-9e075924e32d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.659714557Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.661741511Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.661817733Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.661873724Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.666319269Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.670641203Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:39.714807108Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"uri":"/crucible/0/regions","method":"POST","req_id":"bef67f63-318b-462b-9f6c-bf61c99020ec","remote_addr":"[fd00:1122:3344:101::c]:38903","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.419945921Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"uri":"/crucible/0/regions/764d6ddf-285c-4a2b-8560-9e075924e32d","method":"GET","req_id":"b603b8a1-8baa-40cc-851d-759b75139563","remote_addr":"[fd00:1122:3344:101::c]:38903","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"path is \"/data/regions/764d6ddf-285c-4a2b-8560-9e075924e32d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.422770769Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"datafile"}
{"msg":"dataset is oxp_72ed4737-9cfc-4e16-8b65-a8465f543320/crucible/regions/764d6ddf-285c-4a2b-8560-9e075924e32d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.434672862Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"datafile"}
{"msg":"region 764d6ddf-285c-4a2b-8560-9e075924e32d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.446025325Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.446170878Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-764d6ddf-285c-4a2b-8560-9e075924e32d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.447317507Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.448093841Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"uri":"/crucible/0/regions/764d6ddf-285c-4a2b-8560-9e075924e32d","method":"DELETE","req_id":"f0a82dea-fe12-490b-afd8-381e44631f7b","remote_addr":"[fd00:1122:3344:101::c]:38903","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25354,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.451731643Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"uri":"/crucible/0/regions/764d6ddf-285c-4a2b-8560-9e075924e32d","method":"GET","req_id":"ead991a5-b4dd-4c9c-b7f1-ae90fc3121ba","remote_addr":"[fd00:1122:3344:101::c]:38903","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.453816658Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_72ed4737-9cfc-4e16-8b65-a8465f543320/crucible/regions/764d6ddf-285c-4a2b-8560-9e075924e32d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.464729255Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"region":"764d6ddf-285c-4a2b-8560-9e075924e32d","component":"worker"}
{"msg":"region 764d6ddf-285c-4a2b-8560-9e075924e32d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.528113225Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-17T08:29:40.54305045Z","hostname":"oxz_crucible_ec83aba1-0d3c-4c53-b9d9-b1310c85f51a","pid":22833,"uri":"/crucible/0/regions/764d6ddf-285c-4a2b-8560-9e075924e32d","method":"GET","req_id":"1624d6e4-2268-41f8-8608-7444877ff78e","remote_addr":"[fd00:1122:3344:101::c]:38903","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":78,"response_code":"200"}