[ Oct 19 04:09:00 Enabled. ]
[ Oct 19 04:09:00 Rereading configuration. ]
[ Oct 19 04:09:02 Rereading configuration. ]
[ Oct 19 04:09:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d6a625a4-69d4-4b17-bd2c-41c77098c6fe/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d6a625a4-69d4-4b17-bd2c-41c77098c6fe/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:09:03.817026032Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:09:03.818167005Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:09:03.818179025Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:09:03.836097258Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:09:03.969490765Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.245872487Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:45205"}
{"msg":"region 670ffd24-0921-4f05-8a5c-71d19dcf5e6c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.267171317Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.271555333Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.27546949Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"uri":"/crucible/0/regions","method":"POST","req_id":"b4e091cf-d3d4-4a56-bcf2-e1bf166e07a0","remote_addr":"[fd00:1122:3344:101::b]:45205","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":15014,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_d6a625a4-69d4-4b17-bd2c-41c77098c6fe/crucible/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.282479998Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_d6a625a4-69d4-4b17-bd2c-41c77098c6fe/crucible/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.282501099Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"670ffd24-0921-4f05-8a5c-71d19dcf5e6c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.346333546Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"region":"670ffd24-0921-4f05-8a5c-71d19dcf5e6c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"670ffd24-0921-4f05-8a5c-71d19dcf5e6c\" \"--data\" \"/data/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.348273744Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"region":"670ffd24-0921-4f05-8a5c-71d19dcf5e6c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.453533008Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"uri":"/crucible/0/regions","method":"POST","req_id":"8d9a9e6b-8c9c-46e0-8e44-9c14218cba14","remote_addr":"[fd00:1122:3344:101::b]:45205","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":507,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.461338221Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"region":"670ffd24-0921-4f05-8a5c-71d19dcf5e6c","component":"worker"}
{"msg":"region 670ffd24-0921-4f05-8a5c-71d19dcf5e6c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.461369062Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.461492805Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-670ffd24-0921-4f05-8a5c-71d19dcf5e6c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.461883632Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-670ffd24-0921-4f05-8a5c-71d19dcf5e6c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.465734528Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.479715574Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-670ffd24-0921-4f05-8a5c-71d19dcf5e6c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.483872436Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.494076497Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.496042175Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.496129887Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.496197478Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.501069884Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.505856358Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.843189974Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"uri":"/crucible/0/regions","method":"POST","req_id":"2ce005e7-4f89-4faa-8057-42b99fceb889","remote_addr":"[fd00:1122:3344:101::b]:45205","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.493087566Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"uri":"/crucible/0/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c","method":"GET","req_id":"23f7f07e-dfd1-4acc-8741-4f6d89f8d635","remote_addr":"[fd00:1122:3344:101::b]:45205","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":558,"response_code":"200"}
{"msg":"path is \"/data/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.49481129Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"datafile"}
{"msg":"dataset is oxp_d6a625a4-69d4-4b17-bd2c-41c77098c6fe/crucible/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.504049452Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"datafile"}
{"msg":"region 670ffd24-0921-4f05-8a5c-71d19dcf5e6c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.512195573Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.512342996Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.512410187Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"uri":"/crucible/0/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c","method":"DELETE","req_id":"7ae53bef-96db-46cd-88d8-61c6e604db35","remote_addr":"[fd00:1122:3344:101::b]:45205","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":17631,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-670ffd24-0921-4f05-8a5c-71d19dcf5e6c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.513393826Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.514972667Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"uri":"/crucible/0/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c","method":"GET","req_id":"6a4d75a8-c155-469f-88d6-5665b4248ad1","remote_addr":"[fd00:1122:3344:101::b]:45205","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":29,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.517733932Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d6a625a4-69d4-4b17-bd2c-41c77098c6fe/crucible/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.526856551Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"region":"670ffd24-0921-4f05-8a5c-71d19dcf5e6c","component":"worker"}
{"msg":"region 670ffd24-0921-4f05-8a5c-71d19dcf5e6c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.584489107Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.648263143Z","hostname":"oxz_crucible_c3b8c3d5-e7c1-4fb9-a729-d54836e7a087","pid":21501,"uri":"/crucible/0/regions/670ffd24-0921-4f05-8a5c-71d19dcf5e6c","method":"GET","req_id":"642ee144-3787-4ba7-91b0-5ceb82892169","remote_addr":"[fd00:1122:3344:101::b]:45205","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":54,"response_code":"200"}