Skip to content

Commit c55ac6b

Browse files
coreationcoreation
committed
Merge pull request #380 from tdt/development
Development
2 parents b79658b + 554615c commit c55ac6b

6 files changed

Lines changed: 123 additions & 51 deletions

File tree

‎app/Tdt/Core/DataControllers/CSVController.php‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
use Tdt\Core\Pager;
77
use Tdt\Core\Repositories\Interfaces\TabularColumnsRepositoryInterface;
88
use Tdt\Core\Repositories\Interfaces\GeoPropertyRepositoryInterface;
9+
use Tdt\Core\Datasets\DatasetController;
10+
11+
ini_set('auto_detect_line_endings', true);
912

1013
/**
1114
* CSV Controller
@@ -34,6 +37,15 @@ public function readData($source_definition, $rest_parameters = array())
3437
{
3538
list($limit, $offset) = Pager::calculateLimitAndOffset();
3639

40+
// Get the format, if it's CSV we allow a full read of the datasource without paging limitation
41+
list($uri, $extension) = $this->processURI(\Request::path());
42+
43+
$ignore_paging = false;
44+
45+
if (strtolower($extension) == 'csv') {
46+
$ignore_paging = true;
47+
}
48+
3749
// Disregard the paging when rest parameters are given
3850
if (!empty($rest_parameters)) {
3951
$limit = PHP_INT_MAX;
@@ -130,7 +142,7 @@ public function readData($source_definition, $rest_parameters = array())
130142
);
131143

132144
if (($handle = fopen($uri, "r", false, stream_context_create($ssl_options))) !== false) {
133-
while (($data = fgetcsv($handle, 2000000, $delimiter)) !== false) {
145+
while (($data = fgetcsv($handle, 0, $delimiter)) !== false) {
134146
if ($total_rows >= $start_row) {
135147
// Create the values array, containing the (aliased) name of the column
136148
// to the value of a the row which $data represents
@@ -158,12 +170,11 @@ public function readData($source_definition, $rest_parameters = array())
158170

159171
$total_rows++;
160172

161-
if ($total_rows >= 10000) {
173+
if ($total_rows >= 10000 && !$ignore_paging) {
162174
break;
163175
}
164176
}
165177
fclose($handle);
166-
167178
} else {
168179
\App::abort(500, "Cannot retrieve any data from the CSV file on location $uri.");
169180
}
@@ -199,6 +210,41 @@ private function createValues($columns, $data)
199210
return $result;
200211
}
201212

213+
/**
214+
* Parse the URI and look for the resource name and the format
215+
*
216+
* @param string $uri
217+
*
218+
* @return array
219+
*/
220+
private function processURI($uri)
221+
{
222+
$dot_position = strrpos($uri, '.');
223+
224+
if (!$dot_position) {
225+
return array($uri, null);
226+
}
227+
228+
// If a dot has been found, do a couple
229+
// of checks to find out if it introduces a formatter
230+
$uri_parts = explode('.', $uri);
231+
232+
$possible_extension = strtoupper(array_pop($uri_parts));
233+
234+
$uri = implode('.', $uri_parts);
235+
236+
$formatter_class = 'Tdt\\Core\\Formatters\\' . $possible_extension . 'Formatter';
237+
238+
if (!class_exists($formatter_class)) {
239+
// Re-attach the dot with the latter part of the uri
240+
$uri .= '.' . strtolower($possible_extension);
241+
242+
return array($uri, null);
243+
}
244+
245+
return array($uri, strtolower($possible_extension));
246+
}
247+
202248
/**
203249
* Parse the columns from a CSV file and return them
204250
* Optionally aliases can be given to columns as well as a primary key

‎app/Tdt/Core/Datasets/DatasetController.php‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -280,22 +280,24 @@ private function processURI($uri)
280280
private static function applyRestFilter($data, $rest_params)
281281
{
282282
foreach ($rest_params as $rest_param) {
283-
if (is_object($data) && $key = self::propertyExists($data, $rest_param)) {
284-
$data = $data->$key;
285-
} elseif (is_array($data)) {
286-
if ($key = self::keyExists($data, $rest_param)) {
287-
$data = $data[$key];
288-
} elseif (is_numeric($rest_param)) {
289-
for ($i = 0; $i <= $rest_param; $i++) {
290-
$result = array_shift($data);
291-
}
283+
if (!empty($rest_param)) {
284+
if (is_object($data) && $key = self::propertyExists($data, $rest_param)) {
285+
$data = $data->$key;
286+
} elseif (is_array($data)) {
287+
if ($key = self::keyExists($data, $rest_param)) {
288+
$data = $data[$key];
289+
} elseif (is_numeric($rest_param)) {
290+
for ($i = 0; $i <= $rest_param; $i++) {
291+
$result = array_shift($data);
292+
}
292293

293-
$data = $result;
294+
$data = $result;
295+
} else {
296+
\App::abort(404, "No property ($rest_param) has been found.");
297+
}
294298
} else {
295299
\App::abort(404, "No property ($rest_param) has been found.");
296300
}
297-
} else {
298-
\App::abort(404, "No property ($rest_param) has been found.");
299301
}
300302
}
301303

‎app/Tdt/Core/Repositories/DcatRepository.php‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,6 @@ public function getDcatDocument(array $definitions, $oldest_definition)
3737
// Create a new EasyRDF graph
3838
$graph = new \EasyRdf_Graph();
3939

40-
\EasyRdf_Namespace::set('adms', 'http://www.w3.org/ns/adms#');
41-
4240
$all_settings = $this->settings->getAll();
4341

4442
$uri = \Request::root();
@@ -105,7 +103,7 @@ public function getDcatDocument(array $definitions, $oldest_definition)
105103

106104
// Backwards compatibility
107105
if (!empty($definition['contact_point'])) {
108-
$graph->addResource($dataset_uri, 'adms:contactPoint', $definition['contact_point']);
106+
$graph->addResource($dataset_uri, 'dcat:contactPoint', $definition['contact_point']);
109107
}
110108

111109
// Add the publisher resource to the dataset

‎app/Tdt/Core/Validators/CustomValidator.php‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,18 @@
1010
*/
1111
class CustomValidator extends \Illuminate\Validation\Validator
1212
{
13-
1413
/**
1514
* Check if the URI can be resolved externally or locally
1615
*/
1716
public function validateUri($attribute, $value, $parameters)
1817
{
1918
try {
20-
if (!filter_var($value, FILTER_VALIDATE_URL) === false) {
21-
$data = $this->getRemoteData($value);
19+
$url_pieces = parse_url($value);
2220

23-
return !empty($data);
21+
if (!filter_var($value, FILTER_VALIDATE_URL) === false && ($url_pieces['scheme'] == 'http' || $url_pieces['scheme'] == 'https')) {
22+
$status = $this->getHeadInfo($value);
23+
24+
return $status == 200;
2425
} else {
2526
$data =@ file_get_contents($value);
2627

@@ -31,6 +32,31 @@ public function validateUri($attribute, $value, $parameters)
3132
}
3233
}
3334

35+
private function getHeadInfo($uri)
36+
{
37+
$c = curl_init();
38+
curl_setopt($c, CURLOPT_URL, $uri);
39+
curl_setopt($c, CURLOPT_RETURNTRANSFER, 1);
40+
41+
curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
42+
curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
43+
44+
curl_setopt($c, CURLOPT_CONNECTTIMEOUT, 9);
45+
curl_setopt($c, CURLOPT_TIMEOUT, 60);
46+
curl_setopt($c, CURLOPT_CUSTOMREQUEST, 'HEAD');
47+
curl_setopt($c, CURLOPT_NOBODY, true);
48+
49+
curl_exec($c);
50+
$status = curl_getinfo($c);
51+
curl_close($c);
52+
53+
if (!empty($status['http_code'])) {
54+
return $status['http_code'];
55+
} else {
56+
return 500;
57+
}
58+
}
59+
3460
private function getRemoteData($url)
3561
{
3662
$c = curl_init();

‎composer.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
}
1717
},
1818
"require": {
19-
"laravel/framework": "4.2.0",
19+
"laravel/framework": "4.2.1",
2020
"phpoffice/phpexcel": "1.8.0",
2121
"phpunit/phpunit" : "4.0.17",
2222
"easyrdf/easyrdf" : "0.9.1",

‎composer.lock‎

Lines changed: 28 additions & 28 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)