[ Oct 22 00:32:30 Enabled. ]
[ Oct 22 00:32:30 Rereading configuration. ]
[ Oct 22 00:32:35 Rereading configuration. ]
[ Oct 22 00:32:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_4124bd0f-6d92-452d-b8ec-890efde7c175/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_4124bd0f-6d92-452d-b8ec-890efde7c175/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:32:38.899004112Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:32:38.900849272Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:32:38.900860152Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:32:38.956983303Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:32:39.233483008Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:32.881038334Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:41560"}
{"msg":"region 26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:32.911327435Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:32.91597816Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"uri":"/crucible/0/regions","method":"POST","req_id":"d3622f87-3b7f-43a1-9445-68f9b416e0e5","remote_addr":"[fd00:1122:3344:101::c]:41560","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":12729,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:32.917109329Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_4124bd0f-6d92-452d-b8ec-890efde7c175/crucible/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:32.926691944Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_4124bd0f-6d92-452d-b8ec-890efde7c175/crucible/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:32.926722955Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:32.980313524Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"region":"26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9\" \"--data\" \"/data/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:32.984269859Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"region":"26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.109054802Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"region":"26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","component":"worker"}
{"msg":"region 26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.109105133Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.109222405Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.111836588Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.117044302Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.134552896Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.137637226Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.144841693Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.144903354Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.144948735Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.144988105Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.14837231Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.152193042Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.207982217Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"uri":"/crucible/0/regions","method":"POST","req_id":"57c6be49-4c3e-43e2-b0de-86e4d720e29d","remote_addr":"[fd00:1122:3344:101::c]:41560","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.532283829Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"uri":"/crucible/0/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","method":"GET","req_id":"2a5ae8c0-014f-43f5-a0a0-867a0730b1c2","remote_addr":"[fd00:1122:3344:101::c]:41560","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"path is \"/data/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.535181336Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"datafile"}
{"msg":"dataset is oxp_4124bd0f-6d92-452d-b8ec-890efde7c175/crucible/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.546828285Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"datafile"}
{"msg":"region 26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.556157326Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.556300419Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.55636699Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"uri":"/crucible/0/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","method":"DELETE","req_id":"e201362a-b922-4555-9726-9c0eaecc225d","remote_addr":"[fd00:1122:3344:101::c]:41560","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":21208,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.557321295Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.559319558Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"uri":"/crucible/0/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","method":"GET","req_id":"541f9432-8b2b-4607-955f-d2dd7fae135f","remote_addr":"[fd00:1122:3344:101::c]:41560","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":36,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.562197054Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_4124bd0f-6d92-452d-b8ec-890efde7c175/crucible/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.57304081Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"region":"26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","component":"worker"}
{"msg":"region 26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.635211309Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T00:33:33.642005639Z","hostname":"oxz_crucible_960438c0-57fe-4406-a0be-4f4ee6b84a88","pid":21950,"uri":"/crucible/0/regions/26add2fa-e8a5-4d63-a5af-ca1b5cfb5bd9","method":"GET","req_id":"ec57d0b7-59d2-45d5-ae87-26d9ac65f1f8","remote_addr":"[fd00:1122:3344:101::c]:41560","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":78,"response_code":"200"}