[ Dec 16 20:49:48 Enabled. ]
[ Dec 16 20:49:48 Rereading configuration. ]
[ Dec 16 20:49:49 Rereading configuration. ]
[ Dec 16 20:49:52 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_96bdfb19-1c18-4a31-89a8-33ac935d9bd6/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_96bdfb19-1c18-4a31-89a8-33ac935d9bd6/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:49:52.848932721Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:49:52.854583414Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:49:52.854597505Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:49:52.873028471Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:49:53.125890323Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.441150681Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61330"}
{"msg":"region 34392aac-791b-433d-ad13-41ce7fb5b298 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.479565485Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.486391446Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.492547813Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"uri":"/crucible/0/regions","method":"POST","req_id":"39b30ec5-7a02-4009-9e77-c285ebb7176d","remote_addr":"[fd00:1122:3344:101::c]:61330","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19419,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_96bdfb19-1c18-4a31-89a8-33ac935d9bd6/crucible/regions/34392aac-791b-433d-ad13-41ce7fb5b298","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.499287312Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_96bdfb19-1c18-4a31-89a8-33ac935d9bd6/crucible/regions/34392aac-791b-433d-ad13-41ce7fb5b298","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.499309052Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"34392aac-791b-433d-ad13-41ce7fb5b298\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/34392aac-791b-433d-ad13-41ce7fb5b298\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.545540886Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"region":"34392aac-791b-433d-ad13-41ce7fb5b298","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"34392aac-791b-433d-ad13-41ce7fb5b298\" \"--data\" \"/data/regions/34392aac-791b-433d-ad13-41ce7fb5b298\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.548298168Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"region":"34392aac-791b-433d-ad13-41ce7fb5b298","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.863122954Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"uri":"/crucible/0/regions","method":"POST","req_id":"7ebce8c0-1a10-425f-97fe-e9640fd61d97","remote_addr":"[fd00:1122:3344:101::c]:61330","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.892125618Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"region":"34392aac-791b-433d-ad13-41ce7fb5b298","component":"worker"}
{"msg":"region 34392aac-791b-433d-ad13-41ce7fb5b298 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.892167739Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.892289781Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-34392aac-791b-433d-ad13-41ce7fb5b298","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.892694089Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-34392aac-791b-433d-ad13-41ce7fb5b298","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.900923186Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.914787731Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-34392aac-791b-433d-ad13-41ce7fb5b298","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.917385211Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/34392aac-791b-433d-ad13-41ce7fb5b298","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.917826459Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.91788667Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.917928561Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.917966662Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.923352755Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:40.92729859Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.539684241Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"uri":"/crucible/0/regions","method":"POST","req_id":"456d52b9-f24d-4410-a9cb-c035c59ea182","remote_addr":"[fd00:1122:3344:101::c]:61330","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.821704439Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"uri":"/crucible/0/regions/34392aac-791b-433d-ad13-41ce7fb5b298","method":"GET","req_id":"ed3ab4a9-5933-493f-adaa-e7f886d74492","remote_addr":"[fd00:1122:3344:101::c]:61330","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"path is \"/data/regions/34392aac-791b-433d-ad13-41ce7fb5b298\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.825446111Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"datafile"}
{"msg":"dataset is oxp_96bdfb19-1c18-4a31-89a8-33ac935d9bd6/crucible/regions/34392aac-791b-433d-ad13-41ce7fb5b298","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.836857469Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"datafile"}
{"msg":"region 34392aac-791b-433d-ad13-41ce7fb5b298 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.845981934Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.846142306Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.846208528Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"uri":"/crucible/0/regions/34392aac-791b-433d-ad13-41ce7fb5b298","method":"DELETE","req_id":"8250327c-a68d-4c15-8d85-aaaa0cf3ebcd","remote_addr":"[fd00:1122:3344:101::c]:61330","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20793,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-34392aac-791b-433d-ad13-41ce7fb5b298 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.849507211Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.849724775Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"uri":"/crucible/0/regions/34392aac-791b-433d-ad13-41ce7fb5b298","method":"GET","req_id":"0cd61cdc-c2a1-45c8-aff8-9ef0e4b00ba4","remote_addr":"[fd00:1122:3344:101::c]:61330","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.854115309Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_96bdfb19-1c18-4a31-89a8-33ac935d9bd6/crucible/regions/34392aac-791b-433d-ad13-41ce7fb5b298\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.865337643Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"region":"34392aac-791b-433d-ad13-41ce7fb5b298","component":"worker"}
{"msg":"region 34392aac-791b-433d-ad13-41ce7fb5b298 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.9436633Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T20:50:41.953478257Z","hostname":"oxz_crucible_07e87899-ef0c-472f-aab6-cd322c2647db","pid":21264,"uri":"/crucible/0/regions/34392aac-791b-433d-ad13-41ce7fb5b298","method":"GET","req_id":"b8985648-c68e-4c61-a415-447db800b625","remote_addr":"[fd00:1122:3344:101::c]:61330","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":68,"response_code":"200"}