[ Oct 30 17:49:03 Enabled. ]
[ Oct 30 17:49:03 Rereading configuration. ]
[ Oct 30 17:49:04 Rereading configuration. ]
[ Oct 30 17:49:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7c337fba-b737-4f90-a327-065b4f3b4476/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7c337fba-b737-4f90-a327-065b4f3b4476/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:05.797506981Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:05.799449138Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:05.799459889Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:05.831199418Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:05.983283072Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.388275902Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42590"}
{"msg":"region abd2bda7-8e8f-4ead-a114-fd39a98f5ade state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.418730256Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.422769187Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"uri":"/crucible/0/regions","method":"POST","req_id":"a7694153-9329-46d3-9b86-dc812a042013","remote_addr":"[fd00:1122:3344:101::a]:42590","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":11788,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.423860916Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_7c337fba-b737-4f90-a327-065b4f3b4476/crucible/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.432595731Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_7c337fba-b737-4f90-a327-065b4f3b4476/crucible/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.432622642Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"abd2bda7-8e8f-4ead-a114-fd39a98f5ade\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.491853481Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"region":"abd2bda7-8e8f-4ead-a114-fd39a98f5ade","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"abd2bda7-8e8f-4ead-a114-fd39a98f5ade\" \"--data\" \"/data/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.495809159Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"region":"abd2bda7-8e8f-4ead-a114-fd39a98f5ade","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.589554615Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"uri":"/crucible/0/regions","method":"POST","req_id":"e830aa12-91a5-4701-99a2-3a2ac5ca118b","remote_addr":"[fd00:1122:3344:101::a]:42590","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.60828553Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"region":"abd2bda7-8e8f-4ead-a114-fd39a98f5ade","component":"worker"}
{"msg":"region abd2bda7-8e8f-4ead-a114-fd39a98f5ade state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.608317141Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.608463656Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-abd2bda7-8e8f-4ead-a114-fd39a98f5ade","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.610775067Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-abd2bda7-8e8f-4ead-a114-fd39a98f5ade","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.615598215Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.633687027Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-abd2bda7-8e8f-4ead-a114-fd39a98f5ade","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.637795561Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.642286038Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.6423527Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.642399672Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.642444833Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.646155003Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.65007854Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:08.873403363Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"uri":"/crucible/0/regions","method":"POST","req_id":"f5ba63e1-989c-49e9-b49d-2b8414bcbbbc","remote_addr":"[fd00:1122:3344:101::a]:42590","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.030549885Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"uri":"/crucible/0/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade","method":"GET","req_id":"fb741f24-8474-4a2a-be74-435a40fa86ce","remote_addr":"[fd00:1122:3344:101::a]:42590","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"path is \"/data/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.032953089Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"datafile"}
{"msg":"dataset is oxp_7c337fba-b737-4f90-a327-065b4f3b4476/crucible/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.045662543Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"datafile"}
{"msg":"region abd2bda7-8e8f-4ead-a114-fd39a98f5ade state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.057234637Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.057505097Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.057584009Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"uri":"/crucible/0/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade","method":"DELETE","req_id":"0e282c56-4173-4f1a-8dfb-add3304d8cfb","remote_addr":"[fd00:1122:3344:101::a]:42590","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24649,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-abd2bda7-8e8f-4ead-a114-fd39a98f5ade (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.058643136Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.061345321Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"uri":"/crucible/0/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade","method":"GET","req_id":"0515c1e8-85d9-4170-9118-d10813b14553","remote_addr":"[fd00:1122:3344:101::a]:42590","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":40,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.064832233Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7c337fba-b737-4f90-a327-065b4f3b4476/crucible/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.078668446Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"region":"abd2bda7-8e8f-4ead-a114-fd39a98f5ade","component":"worker"}
{"msg":"region abd2bda7-8e8f-4ead-a114-fd39a98f5ade state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.164536006Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:10.17151359Z","hostname":"oxz_crucible_7180a284-fbab-4f51-8bb9-97f62b818245","pid":20994,"uri":"/crucible/0/regions/abd2bda7-8e8f-4ead-a114-fd39a98f5ade","method":"GET","req_id":"56666a75-58c9-4482-bc81-1785977ea4f2","remote_addr":"[fd00:1122:3344:101::a]:42590","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":74,"response_code":"200"}