[ Mar 18 03:45:30 Enabled. ]
[ Mar 18 03:45:30 Rereading configuration. ]
[ Mar 18 03:45:33 Rereading configuration. ]
[ Mar 18 03:45:37 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:45:37.554197416Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:45:37.556814545Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:45:37.556823065Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:45:37.575697128Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:45:37.705723332Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.155264388Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:44107"}
{"msg":"region beea13e7-66bd-4adc-90e0-a0e00e1b235c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.196580958Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.19936056Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"uri":"/crucible/0/regions","method":"POST","req_id":"d9230aca-3ba7-4231-8654-313ab4a09f57","remote_addr":"[fd00:1122:3344:101::c]:44107","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":17974,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.199382581Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.207910818Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.207936039Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"beea13e7-66bd-4adc-90e0-a0e00e1b235c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.269521254Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"region":"beea13e7-66bd-4adc-90e0-a0e00e1b235c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"beea13e7-66bd-4adc-90e0-a0e00e1b235c\" \"--data\" \"/data/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.271434003Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"region":"beea13e7-66bd-4adc-90e0-a0e00e1b235c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.3624501Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"uri":"/crucible/0/regions","method":"POST","req_id":"d8d777b1-a670-4417-b635-7c5ee3322957","remote_addr":"[fd00:1122:3344:101::c]:44107","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.519097952Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"region":"beea13e7-66bd-4adc-90e0-a0e00e1b235c","component":"worker"}
{"msg":"region beea13e7-66bd-4adc-90e0-a0e00e1b235c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.519141333Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.519238455Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-beea13e7-66bd-4adc-90e0-a0e00e1b235c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.519667271Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-beea13e7-66bd-4adc-90e0-a0e00e1b235c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.523830584Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.536044477Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-beea13e7-66bd-4adc-90e0-a0e00e1b235c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.539654861Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.540372962Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.540435593Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.540479924Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.540522204Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.544743148Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:31.559692502Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:46:32.024771958Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"uri":"/crucible/0/regions","method":"POST","req_id":"dd13d4e1-94a1-447f-b66a-d7d163c8548f","remote_addr":"[fd00:1122:3344:101::c]:44107","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":46,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.627288285Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:63545"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.630428212Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"uri":"/crucible/0/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c","method":"GET","req_id":"095d6b1c-20ae-4093-af4a-8f31be8235c8","remote_addr":"[fd00:1122:3344:101::c]:63545","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":3089,"response_code":"200"}
{"msg":"path is \"/data/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.636032886Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"dataset is oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.647419558Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"snapshot is oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c@03a5489a-fbea-47cc-ad2a-5b741ea09434","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.655493809Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c\", \"03a5489a-fbea-47cc-ad2a-5b741ea09434\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.655522459Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.665365387Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"uri":"/crucible/0/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c/snapshots/03a5489a-fbea-47cc-ad2a-5b741ea09434","method":"GET","req_id":"c59c5b55-6711-430c-9e21-febb3e889d54","remote_addr":"[fd00:1122:3344:101::c]:63545","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":31579,"response_code":"200"}
{"msg":"path is \"/data/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.669115353Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"dataset is oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.677128934Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"snapshot is oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c@03a5489a-fbea-47cc-ad2a-5b741ea09434","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.685746053Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c\", \"03a5489a-fbea-47cc-ad2a-5b741ea09434\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.685792224Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot beea13e7-66bd-4adc-90e0-a0e00e1b235c-03a5489a-fbea-47cc-ad2a-5b741ea09434 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.695048813Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"applying SMF actions for region beea13e7-66bd-4adc-90e0-a0e00e1b235c running snapshot 03a5489a-fbea-47cc-ad2a-5b741ea09434 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.695616751Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.695636912Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"uri":"/crucible/0/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c/snapshots/03a5489a-fbea-47cc-ad2a-5b741ea09434/run","method":"POST","req_id":"e3a5d2a5-7f28-4c21-89bd-172abd6f9177","remote_addr":"[fd00:1122:3344:101::c]:63545","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27067,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-beea13e7-66bd-4adc-90e0-a0e00e1b235c-03a5489a-fbea-47cc-ad2a-5b741ea09434","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.707858825Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-beea13e7-66bd-4adc-90e0-a0e00e1b235c-03a5489a-fbea-47cc-ad2a-5b741ea09434","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.711209856Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-beea13e7-66bd-4adc-90e0-a0e00e1b235c-03a5489a-fbea-47cc-ad2a-5b741ea09434","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.722356863Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c/.zfs/snapshot/03a5489a-fbea-47cc-ad2a-5b741ea09434","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.722670398Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.722723138Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.722766409Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.72280325Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.72285093Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.726220111Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.73017411Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"worker"}
{"msg":"region beea13e7-66bd-4adc-90e0-a0e00e1b235c running snapshot 03a5489a-fbea-47cc-ad2a-5b741ea09434 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:06.730185581Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"path is \"/data/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:07.030045434Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"dataset is oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:07.039858232Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"datafile"}
{"msg":"snapshot is oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c@03a5489a-fbea-47cc-ad2a-5b741ea09434","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:07.047786231Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_539281b1-65d1-48bc-b3ec-a5c252c45150/crucible/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c\", \"03a5489a-fbea-47cc-ad2a-5b741ea09434\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:07.047827231Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-18T03:57:07.055944943Z","hostname":"oxz_crucible_ac9ae711-eec1-410a-89bc-135bc14fe4e9","pid":27102,"uri":"/crucible/0/regions/beea13e7-66bd-4adc-90e0-a0e00e1b235c/snapshots/03a5489a-fbea-47cc-ad2a-5b741ea09434/run","method":"POST","req_id":"b470c1fa-ac55-4614-8b89-23ed7c544903","remote_addr":"[fd00:1122:3344:101::c]:63545","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25927,"response_code":"200"}