[ Apr 29 04:10:32 Enabled. ]
[ Apr 29 04:10:32 Rereading configuration. ]
[ Apr 29 04:10:35 Rereading configuration. ]
[ Apr 29 04:10:39 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_feef97bb-ca23-472f-b173-ce352fd7d46c/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_feef97bb-ca23-472f-b173-ce352fd7d46c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:10:39.17296581Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:10:39.174035832Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:10:39.174049123Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:10:39.191936439Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:10:39.802196834Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.081966105Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:36499"}
{"msg":"region b91694a2-fbf8-47f4-9701-c129cff78b7d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.112118564Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.113272298Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"uri":"/crucible/0/regions","method":"POST","req_id":"a10d6a86-4995-405e-983a-2a53f5e8a7e6","remote_addr":"[fd00:1122:3344:101::c]:36499","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":1893,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.113286938Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_feef97bb-ca23-472f-b173-ce352fd7d46c/crucible/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.123240912Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_feef97bb-ca23-472f-b173-ce352fd7d46c/crucible/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.123292633Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b91694a2-fbf8-47f4-9701-c129cff78b7d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.195306069Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"region":"b91694a2-fbf8-47f4-9701-c129cff78b7d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b91694a2-fbf8-47f4-9701-c129cff78b7d\" \"--data\" \"/data/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.197353621Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"region":"b91694a2-fbf8-47f4-9701-c129cff78b7d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.311794868Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"region":"b91694a2-fbf8-47f4-9701-c129cff78b7d","component":"worker"}
{"msg":"region b91694a2-fbf8-47f4-9701-c129cff78b7d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.31186127Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.312084604Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b91694a2-fbf8-47f4-9701-c129cff78b7d","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.312522743Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b91694a2-fbf8-47f4-9701-c129cff78b7d","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.318257851Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.326873258Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b91694a2-fbf8-47f4-9701-c129cff78b7d","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.32945048Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.330008672Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.330104394Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.330143774Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.330187755Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.336112287Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.340480017Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:07.401737693Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"uri":"/crucible/0/regions","method":"POST","req_id":"7310dc14-fcec-4f83-86e1-d1f5446ef107","remote_addr":"[fd00:1122:3344:101::c]:36499","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":827,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.050352975Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:62811"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.050450526Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"uri":"/crucible/0/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d","method":"GET","req_id":"d1847882-c624-49ee-ba9a-5a28d5048a63","remote_addr":"[fd00:1122:3344:101::c]:62811","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"path is \"/data/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.0506093Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"datafile"}
{"msg":"dataset is oxp_feef97bb-ca23-472f-b173-ce352fd7d46c/crucible/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.060304859Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"datafile"}
{"msg":"region b91694a2-fbf8-47f4-9701-c129cff78b7d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.069216311Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.069396495Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.069474057Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"uri":"/crucible/0/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d","method":"DELETE","req_id":"f191aec6-2aeb-40a1-b137-56439589a729","remote_addr":"[fd00:1122:3344:101::c]:62811","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":18897,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.06962868Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"uri":"/crucible/0/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d","method":"GET","req_id":"d835bfc3-9a96-4583-a773-1df731c84d31","remote_addr":"[fd00:1122:3344:101::c]:62811","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":21,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-b91694a2-fbf8-47f4-9701-c129cff78b7d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.070495718Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.074461409Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_feef97bb-ca23-472f-b173-ce352fd7d46c/crucible/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.084605387Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"region":"b91694a2-fbf8-47f4-9701-c129cff78b7d","component":"worker"}
{"msg":"region b91694a2-fbf8-47f4-9701-c129cff78b7d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.142109646Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T04:11:08.19399172Z","hostname":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","pid":23477,"uri":"/crucible/0/regions/b91694a2-fbf8-47f4-9701-c129cff78b7d","method":"GET","req_id":"74806cd6-a92e-45b1-ac27-b8916112a282","remote_addr":"[fd00:1122:3344:101::c]:62811","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":68,"response_code":"200"}