[ Jun 28 23:40:37 Enabled. ]
[ Jun 28 23:40:37 Rereading configuration. ]
[ Jun 28 23:40:39 Rereading configuration. ]
[ Jun 28 23:40:45 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_12882c07-c253-4405-9cb3-95dd30ae3d50/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_12882c07-c253-4405-9cb3-95dd30ae3d50/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:40:45.699946666Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:40:45.703791238Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:40:45.703805588Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:40:45.718273674Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:40:45.91053105Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.655615004Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:39385"}
{"msg":"region 9e8e53b4-c4a7-46bf-82a0-350ccc845683 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.656903835Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.657857087Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"uri":"/crucible/0/regions","method":"POST","req_id":"d9acccbe-ef4f-4160-a990-422759a42ad5","remote_addr":"[fd00:1122:3344:101::c]:39385","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":1263,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.657873508Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_12882c07-c253-4405-9cb3-95dd30ae3d50/crucible/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.668721137Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_12882c07-c253-4405-9cb3-95dd30ae3d50/crucible/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.668783068Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9e8e53b4-c4a7-46bf-82a0-350ccc845683\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.823267912Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"region":"9e8e53b4-c4a7-46bf-82a0-350ccc845683","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9e8e53b4-c4a7-46bf-82a0-350ccc845683\" \"--data\" \"/data/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:12.825208598Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"region":"9e8e53b4-c4a7-46bf-82a0-350ccc845683","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.005644991Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"uri":"/crucible/0/regions","method":"POST","req_id":"88ed335c-5287-468f-9538-777d701243cd","remote_addr":"[fd00:1122:3344:101::c]:39385","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.068706928Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"region":"9e8e53b4-c4a7-46bf-82a0-350ccc845683","component":"worker"}
{"msg":"region 9e8e53b4-c4a7-46bf-82a0-350ccc845683 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.068795981Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.069028336Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9e8e53b4-c4a7-46bf-82a0-350ccc845683","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.069513818Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9e8e53b4-c4a7-46bf-82a0-350ccc845683","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.086583896Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.137829291Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9e8e53b4-c4a7-46bf-82a0-350ccc845683","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.158031324Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.158866384Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.158941825Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.158982077Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.159024018Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.169360365Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.19092472Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:13.280606363Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"uri":"/crucible/0/regions","method":"POST","req_id":"cd317992-b11e-424c-8769-78d68aaa68f7","remote_addr":"[fd00:1122:3344:101::c]:39385","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":578,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.145289573Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"uri":"/crucible/0/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683","method":"GET","req_id":"b337d47c-ac64-4551-a4be-e4caec6c96ae","remote_addr":"[fd00:1122:3344:101::c]:39385","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"path is \"/data/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.147456924Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"datafile"}
{"msg":"dataset is oxp_12882c07-c253-4405-9cb3-95dd30ae3d50/crucible/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.156681275Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"datafile"}
{"msg":"region 9e8e53b4-c4a7-46bf-82a0-350ccc845683 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.165621228Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.165863144Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.165929546Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"uri":"/crucible/0/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683","method":"DELETE","req_id":"aa13326c-3cfc-4f9e-abf6-22b3d15d9d62","remote_addr":"[fd00:1122:3344:101::c]:39385","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":18545,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-9e8e53b4-c4a7-46bf-82a0-350ccc845683 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.167073823Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.168480417Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"uri":"/crucible/0/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683","method":"GET","req_id":"c7bd2d08-8104-47c8-87a1-879acd7ffb45","remote_addr":"[fd00:1122:3344:101::c]:39385","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.180289959Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_12882c07-c253-4405-9cb3-95dd30ae3d50/crucible/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.190134434Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"region":"9e8e53b4-c4a7-46bf-82a0-350ccc845683","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.300921812Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"uri":"/crucible/0/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683","method":"GET","req_id":"fb112d5f-ef20-41ea-8df1-e8cb5dbe7c19","remote_addr":"[fd00:1122:3344:101::c]:39385","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":488,"response_code":"200"}
{"msg":"region 9e8e53b4-c4a7-46bf-82a0-350ccc845683 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.30335982Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T23:41:14.403616217Z","hostname":"oxz_crucible_a62767b3-b6c5-4425-9722-707797a00dc2","pid":25137,"uri":"/crucible/0/regions/9e8e53b4-c4a7-46bf-82a0-350ccc845683","method":"GET","req_id":"2765724b-c05b-4446-94df-88f3aadafb88","remote_addr":"[fd00:1122:3344:101::c]:39385","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":41,"response_code":"200"}