[ Jan 29 21:06:20 Enabled. ]
[ Jan 29 21:06:20 Rereading configuration. ]
[ Jan 29 21:06:21 Rereading configuration. ]
[ Jan 29 21:06:26 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_539af514-3197-4c1e-821e-48e72e0e150f/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_539af514-3197-4c1e-821e-48e72e0e150f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:26.619190113Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:26.622582514Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:26.622592264Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:26.641690932Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:26.762593888Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:45.930226847Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:55479"}
{"msg":"region 0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:45.942292601Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:45.943327424Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"uri":"/crucible/0/regions","method":"POST","req_id":"ff2a8641-c8d8-44f6-9be1-695a62e5dfe4","remote_addr":"[fd00:1122:3344:101::b]:55479","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":3499,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:45.945904454Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_539af514-3197-4c1e-821e-48e72e0e150f/crucible/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:45.956332719Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_539af514-3197-4c1e-821e-48e72e0e150f/crucible/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:45.95639919Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.048224798Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"region":"0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681\" \"--data\" \"/data/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.048316509Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"region":"0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.118746372Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"uri":"/crucible/0/regions","method":"POST","req_id":"3630dc98-01d3-4a36-86ca-1dc83e1bd133","remote_addr":"[fd00:1122:3344:101::b]:55479","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.364568091Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"region":"0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","component":"worker"}
{"msg":"region 0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.364643132Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.364856554Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.365281029Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.37287646Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.395170377Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.404140944Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.404789912Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.407582755Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.407655716Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.407701697Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.433348823Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.443765918Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:46.863994144Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"uri":"/crucible/0/regions","method":"POST","req_id":"48d21c5b-31b9-4b7b-90fc-e0aecf6dfc58","remote_addr":"[fd00:1122:3344:101::b]:55479","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.49087748Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"uri":"/crucible/0/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","method":"GET","req_id":"017e2df0-9aff-47a1-9908-8f5aa24d206a","remote_addr":"[fd00:1122:3344:101::b]:55479","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":49,"response_code":"200"}
{"msg":"path is \"/data/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.494969079Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"datafile"}
{"msg":"dataset is oxp_539af514-3197-4c1e-821e-48e72e0e150f/crucible/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.507676361Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"datafile"}
{"msg":"region 0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.518630122Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.518879175Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.519986269Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.521713949Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"uri":"/crucible/0/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","method":"DELETE","req_id":"be55a224-7fe3-42f4-be2d-9994be7fb8e0","remote_addr":"[fd00:1122:3344:101::b]:55479","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":26789,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.52513276Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"uri":"/crucible/0/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","method":"GET","req_id":"aaf481c5-e198-4d0e-bf59-12ddc38616cf","remote_addr":"[fd00:1122:3344:101::b]:55479","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.537997734Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_539af514-3197-4c1e-821e-48e72e0e150f/crucible/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.550045169Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"region":"0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.616890388Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"uri":"/crucible/0/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","method":"GET","req_id":"c153ff2e-833a-4d8a-8555-f7a6c0839d7b","remote_addr":"[fd00:1122:3344:101::b]:55479","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region 0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.66976286Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T21:06:47.706266236Z","hostname":"oxz_crucible_bfb6e0fe-031b-4658-b79b-316328ed6578","pid":23768,"uri":"/crucible/0/regions/0c8d2cbd-bcea-4cdb-bf00-586a7ef4b681","method":"GET","req_id":"e856f4db-c3a0-4889-8dd5-e45f5adabe90","remote_addr":"[fd00:1122:3344:101::b]:55479","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":42,"response_code":"200"}