[ Oct 19 04:09:00 Enabled. ]
[ Oct 19 04:09:00 Rereading configuration. ]
[ Oct 19 04:09:01 Rereading configuration. ]
[ Oct 19 04:09:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d4b5557b-f3db-4c39-a8ed-f7304d32ea11/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d4b5557b-f3db-4c39-a8ed-f7304d32ea11/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:09:03.708388302Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:09:03.711621306Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:09:03.711634356Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:09:03.756746484Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:09:03.909365151Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.245657143Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:43836"}
{"msg":"region bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.268458052Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.276224375Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.280302125Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"uri":"/crucible/0/regions","method":"POST","req_id":"dd7ec089-b2f8-462c-8514-fb0486436b0e","remote_addr":"[fd00:1122:3344:101::b]:43836","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19097,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_d4b5557b-f3db-4c39-a8ed-f7304d32ea11/crucible/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.288300483Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_d4b5557b-f3db-4c39-a8ed-f7304d32ea11/crucible/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.288322513Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.343146943Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"region":"bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6\" \"--data\" \"/data/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.34552736Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"region":"bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.463105426Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"uri":"/crucible/0/regions","method":"POST","req_id":"4ef9bb19-bafa-4371-a168-5a59f615139b","remote_addr":"[fd00:1122:3344:101::b]:43836","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.59222734Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"region":"bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","component":"worker"}
{"msg":"region bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.592277841Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.592368483Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.592769881Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.595988784Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.61301354Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.62367759Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.626465605Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.628987564Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.629053436Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.629112727Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.632781279Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.637346669Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:09.874855658Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"uri":"/crucible/0/regions","method":"POST","req_id":"91b95a9c-01d7-4301-a8fb-2bd64ba3ef52","remote_addr":"[fd00:1122:3344:101::b]:43836","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":955,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.13542214Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"uri":"/crucible/0/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","method":"GET","req_id":"70d02e75-1f89-449a-bfb1-5fff6716bcef","remote_addr":"[fd00:1122:3344:101::b]:43836","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"path is \"/data/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.137296987Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"datafile"}
{"msg":"dataset is oxp_d4b5557b-f3db-4c39-a8ed-f7304d32ea11/crucible/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.147429667Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"datafile"}
{"msg":"region bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.155982936Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.15619002Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.156264111Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"uri":"/crucible/0/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","method":"DELETE","req_id":"33a523a7-0a7c-4288-8aa0-6e445f0ece67","remote_addr":"[fd00:1122:3344:101::b]:43836","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19004,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.157620878Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.158914553Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"uri":"/crucible/0/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","method":"GET","req_id":"7b5b9c33-5cee-43d3-868e-dcf991d2f504","remote_addr":"[fd00:1122:3344:101::b]:43836","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":628,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.161614376Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d4b5557b-f3db-4c39-a8ed-f7304d32ea11/crucible/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.170777637Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"region":"bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","component":"worker"}
{"msg":"region bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.219508237Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-19T04:10:10.296979493Z","hostname":"oxz_crucible_2d75e171-a583-4bed-b3f1-1dcb4b20ca31","pid":21380,"uri":"/crucible/0/regions/bb83dc6a-4aaf-4ad3-a5e8-5d84243b94b6","method":"GET","req_id":"49f4bc60-f386-415e-a83e-26f9226a56b1","remote_addr":"[fd00:1122:3344:101::b]:43836","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":70,"response_code":"200"}