[ Jul 25 02:08:49 Enabled. ]
[ Jul 25 02:08:49 Rereading configuration. ]
[ Jul 25 02:08:52 Rereading configuration. ]
[ Jul 25 02:08:55 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_07a306b8-24ae-4dbc-9694-d9d608dbd480/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_07a306b8-24ae-4dbc-9694-d9d608dbd480/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:08:55.209812227Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:08:55.214251535Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:08:55.214267925Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:08:55.234413246Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:08:55.480282646Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.250718543Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:53904"}
{"msg":"region 6a090254-b06a-493c-aa12-591d7db26be5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.25964153Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.264474655Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"uri":"/crucible/0/regions","method":"POST","req_id":"b452e6bc-9291-4e3f-a55a-08fafa853af0","remote_addr":"[fd00:1122:3344:101::b]:53904","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":8998,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.264501025Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_07a306b8-24ae-4dbc-9694-d9d608dbd480/crucible/regions/6a090254-b06a-493c-aa12-591d7db26be5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.276902836Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_07a306b8-24ae-4dbc-9694-d9d608dbd480/crucible/regions/6a090254-b06a-493c-aa12-591d7db26be5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.276935297Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6a090254-b06a-493c-aa12-591d7db26be5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6a090254-b06a-493c-aa12-591d7db26be5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.352725065Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"region":"6a090254-b06a-493c-aa12-591d7db26be5","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6a090254-b06a-493c-aa12-591d7db26be5\" \"--data\" \"/data/regions/6a090254-b06a-493c-aa12-591d7db26be5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.354642345Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"region":"6a090254-b06a-493c-aa12-591d7db26be5","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.45095919Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"uri":"/crucible/0/regions","method":"POST","req_id":"f1ba28b2-274f-4a25-86ca-2f3c937d155d","remote_addr":"[fd00:1122:3344:101::b]:53904","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.475334475Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"region":"6a090254-b06a-493c-aa12-591d7db26be5","component":"worker"}
{"msg":"region 6a090254-b06a-493c-aa12-591d7db26be5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.475366566Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.475477008Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6a090254-b06a-493c-aa12-591d7db26be5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.475888554Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6a090254-b06a-493c-aa12-591d7db26be5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.483452371Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.503411578Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6a090254-b06a-493c-aa12-591d7db26be5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.507627683Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6a090254-b06a-493c-aa12-591d7db26be5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.508343964Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.512197583Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.512254544Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.512302425Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.518597552Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.525483028Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:52.731110778Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"uri":"/crucible/0/regions","method":"POST","req_id":"e36db7fb-91f8-4393-a5e7-131b477f47f7","remote_addr":"[fd00:1122:3344:101::b]:53904","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.471853916Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"uri":"/crucible/0/regions/6a090254-b06a-493c-aa12-591d7db26be5","method":"GET","req_id":"7efc68a5-fe60-41c2-9bd9-71ee66c2262f","remote_addr":"[fd00:1122:3344:101::b]:53904","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"path is \"/data/regions/6a090254-b06a-493c-aa12-591d7db26be5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.473745166Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"datafile"}
{"msg":"dataset is oxp_07a306b8-24ae-4dbc-9694-d9d608dbd480/crucible/regions/6a090254-b06a-493c-aa12-591d7db26be5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.482676793Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"datafile"}
{"msg":"region 6a090254-b06a-493c-aa12-591d7db26be5 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.491403538Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.49152986Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.491593621Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"uri":"/crucible/0/regions/6a090254-b06a-493c-aa12-591d7db26be5","method":"DELETE","req_id":"1f0497b7-ebad-40af-8569-36e3c555c750","remote_addr":"[fd00:1122:3344:101::b]:53904","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":17914,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-6a090254-b06a-493c-aa12-591d7db26be5 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.492669447Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.494848901Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"uri":"/crucible/0/regions/6a090254-b06a-493c-aa12-591d7db26be5","method":"GET","req_id":"4e935e64-0487-4465-adce-615a9aa86467","remote_addr":"[fd00:1122:3344:101::b]:53904","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":496,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.496821751Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_07a306b8-24ae-4dbc-9694-d9d608dbd480/crucible/regions/6a090254-b06a-493c-aa12-591d7db26be5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.507197441Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"region":"6a090254-b06a-493c-aa12-591d7db26be5","component":"worker"}
{"msg":"region 6a090254-b06a-493c-aa12-591d7db26be5 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.560706596Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:09:53.633577889Z","hostname":"oxz_crucible_dab49687-4a1d-454d-86fa-c5dbd1f53b48","pid":21794,"uri":"/crucible/0/regions/6a090254-b06a-493c-aa12-591d7db26be5","method":"GET","req_id":"85730900-3b8e-4507-a2e8-b1a6ee41803e","remote_addr":"[fd00:1122:3344:101::b]:53904","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":76,"response_code":"200"}