Added the new mallocWithFallback function which tries to allocate the specified size and falls back to a smaller size when fails

src/app/FTPClient.h

@@ -29,8 +29,8 @@ class FTPClient : public TCPClient
     char *_username = NULL;
     char *_currentDirectory = NULL;
     char *_currentFile = NULL;
-    uint64_t _fileSentBytes = 0;
+    size_t _fileSentBytes = 0;
-    uint64_t _fileRecvBytes = 0;
+    size_t _fileRecvBytes = 0;
     boolean _waitingForDataConnection = false;
     boolean _fileIsBeeingReceived = false;
     uint64_t _actionTimeout = 0;

src/app/FTPServer.h

@@ -6,7 +6,7 @@
 #include "utilities.h"
 #include "Dictionary.h"
 //#define DEBUG_FTPS
-#define READ_BUFFER_SIZE 2048 //2048 is max to read sd card, more will crash
+#define SEND_RECV_BUFFER_SIZE 5000
 
 template <typename T>
 class FTPServer : public TCPServer<T>
@@ -14,9 +14,9 @@ class FTPServer : public TCPServer<T>
   public:
     enum FTPClientState {INIT, WAITING_FOR_COMMANDS};
     enum FTPClientDataTransfer {NONE = 0, LIST_DF, NLST_DF, RETR_DF, STOR_DF, APPE_DF};
-    enum FileTransferStatus {OK, NOT_FOUND, NO_FILE_NAME};
+    enum FileTransferStatus {OK, NOT_FOUND, NO_FILE_NAME, MALLOC_ERROR};
     enum BinaryFlag {OFF = 0, ON};
-    enum FtpMsgCode {_150, _200, _215, _220, _221, _230, _226, _227, _250, _257, _331, _350, _451, _5_502, _504, _530, _550 };
+    enum FTPMsgCode {_150, _200, _215, _220, _221, _230, _226, _227, _250, _257, _331, _350, _451, _5_502, _504, _530, _550 };
 
     FTPServer(uint16_t port = 21, SDClass *sdClass = NULL, const char *login = NULL, const char *password = NULL, uint8_t maxClient = MAX_CLIENT, uint16_t clientCommandDataBufferSize = 255) : TCPServer<T>(port, maxClient, clientCommandDataBufferSize),
       _login(NULL),
@@ -140,7 +140,7 @@ class FTPServer : public TCPServer<T>
               client->_client.println("150 File status okay.");
 
             FileTransferStatus fts;
-            if(!sendFile(client,&fts))//File was sent or error occured
+            if(!sendFile(client,&fts))//Whole file was sent or error an occured
             {
               //we check the return code
               if(fts == OK)
@@ -155,6 +155,12 @@ class FTPServer : public TCPServer<T>
                 client->closeDataConnection();
                 client->_dataTransferPending = NONE;
               }
+              else if(fts == MALLOC_ERROR)
+              {
+                client->_client.println("451 Insufficient RAM space.");
+                client->closeDataConnection();
+                client->_dataTransferPending = NONE;
+              }
             }
           }
         else if(client->_fileSentBytes != 0)
@@ -181,6 +187,14 @@ class FTPServer : public TCPServer<T>
                 {
                   client->_client.println("501 No file name given.");
                 }
+                else if(fts == NOT_FOUND)
+                {
+                  client->_client.println("451 File not found.");
+                }
+                else if(fts == MALLOC_ERROR)
+                {
+                  client->_client.println("451 Insufficient RAM space.");
+                }
                 else
                 {
                   client->_client.println("451 Requested action aborted: local error in processing.");
@@ -238,6 +252,14 @@ class FTPServer : public TCPServer<T>
               {
                 client->_client.println("501 No file name given.");
               }
+              else if(fts == NOT_FOUND)
+              {
+                client->_client.println("451 File not found.");
+              }
+              else if(fts == MALLOC_ERROR)
+              {
+                client->_client.println("451 Insufficient RAM space.");
+              }
               else
               {
                 client->_client.println("451 Requested action aborted: local error in processing.");
@@ -397,23 +419,7 @@ class FTPServer : public TCPServer<T>
           return;
         }
         
-        //We need to test if we are in softAP or STA to chose the right IP
+        client->_client.printf("227 Entering Passive Mode (%u,%u,%u,%u,%d,%d).\r\n", client->_client.localIP()[0], client->_client.localIP()[1], client->_client.localIP()[2], client->_client.localIP()[3], _dataPort / 256, _dataPort % 256);
-        uint8_t addr[4] = {0};
-        if(WiFi.status() == WL_CONNECTED)
-        {
-          addr[0] = WiFi.localIP()[0];
-          addr[1] = WiFi.localIP()[1];
-          addr[2] = WiFi.localIP()[2];
-          addr[3] = WiFi.localIP()[3];
-        }else
-        {
-          addr[0] = WiFi.softAPIP()[0];
-          addr[1] = WiFi.softAPIP()[1];
-          addr[2] = WiFi.softAPIP()[2];
-          addr[3] = WiFi.softAPIP()[3];
-        }
-        
-        client->_client.printf("227 Entering Passive Mode (%u,%u,%u,%u,%d,%d).\r\n", addr[0], addr[1], addr[2], addr[3], _dataPort / 256, _dataPort % 256);
         #ifdef DEBUG_FTPS
         Serial.println("Opening data server for new data client");
         #endif
@@ -803,13 +809,22 @@ class FTPServer : public TCPServer<T>
 
         if(fileBeeingReceived)
         {
-          uint8_t recvBuffer[2048];
+          size_t mallocAcceptedSize(0);
+          uint8_t *recvBuffer = (uint8_t*)mallocWithFallback(SEND_RECV_BUFFER_SIZE * sizeof(uint8_t), &mallocAcceptedSize);
+
+          if(!recvBuffer)
+          {
+            *fts = MALLOC_ERROR;
+            fileBeeingReceived.close();
+            return false;
+          }
 
           /*fileBeeingReceived.seek(client->_fileRecvBytes);*/
-          uint16_t size = client->_dataClient.read(recvBuffer, 2048);
+          size_t size = client->_dataClient.read(recvBuffer, mallocAcceptedSize);
-
           fileBeeingReceived.write(recvBuffer, size);
 
+          free(recvBuffer);
+
           client->_fileRecvBytes += size;
           fileBeeingReceived.close();
           #ifdef DEBUG_FTPS
@@ -818,6 +833,7 @@ class FTPServer : public TCPServer<T>
         }
         else
         {
+          *fts = NOT_FOUND;
           return false;
         }
       }
@@ -840,7 +856,7 @@ class FTPServer : public TCPServer<T>
         File currentDirectory = _sdClass->open(client->_currentDirectory);
         if (currentDirectory)
         {
-          currentDirectory.rewindDirectory();
+          //currentDirectory.rewindDirectory();
           while (true) //May be removed in the future to improve responsiveness
           {
             File fileOrDir = currentDirectory.openNextFile();
@@ -890,26 +906,37 @@ class FTPServer : public TCPServer<T>
     //The binary flag needs to be taken into consideration
     IRAM_ATTR inline boolean sendFile(T *client, FileTransferStatus *fts)
     {
-      if (client->_currentFile != NULL)
+      if (client->_currentFile != nullptr)
       {
-        uint8_t sendBuffer[READ_BUFFER_SIZE];
         File fileToSend = _sdClass->open(client->_currentFile);
         
         if (fileToSend)
         {
           *fts = OK;
-          unsigned int readBytes(0);
+          size_t readBytes(0);
           fileToSend.seek(client->_fileSentBytes);
           
           if(fileToSend.available())
           {
-            readBytes = fileToSend.read(sendBuffer, READ_BUFFER_SIZE);
+            size_t mallocAcceptedSize(0);
+            uint8_t *sendBuffer = (uint8_t*)mallocWithFallback(SEND_RECV_BUFFER_SIZE * sizeof(uint8_t), &mallocAcceptedSize);
+
+            if(!sendBuffer)
+            {
+              *fts = MALLOC_ERROR;
+              fileToSend.close();
+              return false;
+            }
+
+            readBytes = fileToSend.read(sendBuffer, mallocAcceptedSize);
             client->_dataClient.write(sendBuffer, readBytes);
 
+            free(sendBuffer);
+
             client->_fileSentBytes += readBytes;
 
             #ifdef DEBUG_FTPS
-            Serial.printf("File : bytes sent : %u\n",readBytes);
+            Serial.printf("File : bytes sent : %u - free stack : %u\n", readBytes, ESP.getFreeContStack());
             #endif
           }
           else //The whole file has been sent
@@ -978,7 +1005,7 @@ class FTPServer : public TCPServer<T>
     }
 
     //Error code functions
-    void sendInfoResponse(FtpMsgCode code, T *client, const char *msg = "")
+    void sendInfoResponse(FTPMsgCode code, T *client, const char *msg = "")
     {
       switch(code)
       {

src/app/WEBClient.h

@@ -16,7 +16,7 @@ class WEBClient : public TCPClient
 	private:
     WEBServer<WEBClient>::HttpRequestData _httpRequestData;
     WEBServer<WEBClient>::HttpParserStatus _httpParserState = WEBServer<WEBClient>::HttpParserStatus::PARSE_HTTP_VERB;
-    uint64_t _fileSentBytes = 0;
+    size_t _fileSentBytes = 0;
     struct 
     {
       bool _rangeRequest;

src/app/WEBServer.h

@@ -7,7 +7,7 @@
 #include "HttpConstants.h"
 #include "utilities.h"
 //#define DEBUG_WEBS
-#define READ_WRITE_BUFFER_SIZE 2000
+#define READ_WRITE_BUFFER_SIZE 5000
 
 template <typename T>
 class WEBServer : public TCPServer<T>, public HttpConstants
@@ -562,8 +562,7 @@ class WEBServer : public TCPServer<T>, public HttpConstants
       {
         File pageToSend;
         char *filePath(NULL), *header(NULL);
-        uint8_t sendBuffer[READ_WRITE_BUFFER_SIZE];
+        size_t readBytes(0);
-        int readBytes(0);
         //We check what kind of http verb it is
         switch(client->_httpRequestData.HRM)
         {
@@ -576,8 +575,7 @@ class WEBServer : public TCPServer<T>, public HttpConstants
             }
             
             #ifdef DEBUG_WEBS
-            Serial.print("FILE PATH : ");
+            Serial.printf("File path : #%s#\n",filePath);
-            Serial.println(filePath);
             #endif
             
             pageToSend = _sdClass->open(filePath);
@@ -588,7 +586,7 @@ class WEBServer : public TCPServer<T>, public HttpConstants
             {
               char *response(NULL);
               
-              response = (char *) malloc(sizeof(char) * (strlen_P((PGM_P)F("Resource :  not found on this server")) + strlen(client->_httpRequestData.httpResource) + 1));
+              response = (char *) malloc(sizeof(char) * (36 + strlen(client->_httpRequestData.httpResource) + 1));
               if(response == NULL)
               {
                 sendInfoResponse(HTTP_CODE::HTTP_CODE_INTERNAL_SERVER_ERROR, client, "Failed to allocate memory for the response");
@@ -611,10 +609,28 @@ class WEBServer : public TCPServer<T>, public HttpConstants
 
             if(pageToSend.isDirectory()) //TODO : List the files present in the directory
             {
-              //sendDirectoryListing(client, pageToSend); //Sends the content of the directory like what apache does by default. CRASHING FOR NOW, needs to be checked further.
+              //If the ressource wasn't terminated by a '/' then we issue a 301 Moved Permanently, else all's good
+              if(client->_httpRequestData.httpResource[strlen(client->_httpRequestData.httpResource)-1] != '/')
+              {
+                uint16_t locationLength(strlen(client->_httpRequestData.httpResource) + 7/*http://*/ + 16/*ip addr + :*/ + 5/*port*/ + 2);
+                char *location = (char *)malloc(locationLength * sizeof(char));
+                char *message = (char *)malloc((27 + locationLength + 1) * sizeof(char));
+                if(!location || !message)
+                {
+                  sendInfoResponse(HTTP_CODE::HTTP_CODE_INTERNAL_SERVER_ERROR, client, "Failed to allocate memory for the location or message");
                   pageToSend.close();
+                  return false;
+                }
+                sprintf(location, "http://%s:%u%s/",client->_client.localIP().toString().c_str(), TCPServer<T>::getPort(), client->_httpRequestData.httpResource);
+                sprintf(message, "The document has moved to %s.", location);
+                sendInfoResponse(HTTP_CODE::HTTP_CODE_MOVED_PERMANENTLY, client, message, location);
 
-              sendInfoResponse(HTTP_CODE::HTTP_CODE_FORBIDDEN, client, "The file you want to access is a folder");
+                free(location);free(message);
+              }
+              else
+                sendDirectoryListing(client, pageToSend); //Sends the content of the directory like what apache does by default.
+              
+              pageToSend.close();
               return false;
             }
 
@@ -667,12 +683,32 @@ class WEBServer : public TCPServer<T>, public HttpConstants
             
             if(pageToSend.available() && client->_fileSentBytes < client->_rangeData._rangeEnd + 1) // File is done sending once the whole file was sent or the partial part is done sending
             {
-              readBytes = pageToSend.read(sendBuffer,READ_WRITE_BUFFER_SIZE);
+              size_t mallocAcceptedSize(0);
+              /*
+                The original issue was that a buffer of 2048 bytes was allocated on the stack causing a hard to track down stack overflow.
+                Possible solutions I cam up with : 
+                1) Create a statically allocated buffer which would live in the BSS (Block Started by a Symbol) RAM segment.
+                  Advantage : buffer is allocated once and ready to be used immediately.
+                  Drawback : takes space in RAM (lost space) even if the buffer isn't used.
+                2) Create a dynamically allocated buffer using malloc and friends which would live in the heap RAM segment - SOLUTION I IMPLEMENTED
+                  Advantage : buffer is taking RAM only when needed and can be freed afterwards.
+                  Drawback : Allocating and deallocating heap memory a lot is costly in MCU time, leads to RAM fragmentation and could potentially fail...
+              */
+              uint8_t *sendBuffer = (uint8_t*)mallocWithFallback(READ_WRITE_BUFFER_SIZE * sizeof(uint8_t), &mallocAcceptedSize);
+
+              if(!sendBuffer)
+              {
+                pageToSend.close();
+                return false; //Not clean but what else can I do. Should never happen anyway.
+              }
+              
+              readBytes = pageToSend.read(sendBuffer,mallocAcceptedSize);
               client->_client.write(sendBuffer, readBytes);
 
+              free(sendBuffer);
+
               #ifdef DEBUG_WEBS
-              Serial.print("BYTES SENT : ");
+              Serial.printf("Bytes sent : %u, got allocated buffer size : %u - free stack : %u\n", readBytes, mallocAcceptedSize, ESP.getFreeContStack());
-              Serial.println(readBytes);
               #endif
 
               client->_fileSentBytes += readBytes; //We save the number of bytes sent so that we can reopen the file to this position later on.
@@ -702,7 +738,7 @@ class WEBServer : public TCPServer<T>, public HttpConstants
 
     void sendDirectoryListing(T *client, File& pageToSend)
     {
-      /*sendHTTPHeader(&client->_client, HttpConstants::httpMIMETypeToString(HttpConstants::TEXT_HTML));
+      sendHTTPHeader(&client->_client, HttpConstants::httpMIMETypeToString(HttpConstants::TEXT_HTML));
       client->_client.printf_P(PSTR( "<!DOCTYPE HTML>\r\n\
                                       <html>\r\n\
                                         <head>\r\n\
@@ -711,41 +747,66 @@ class WEBServer : public TCPServer<T>, public HttpConstants
                                         <body>\r\n\
                                           <h1>Index of %s</h1>\r\n\
                                           <table>\r\n\
-                                            <tr><th>Name</th><th>Created</th><th>Last modified</th><th>Size</th></tr>\r\n\
+                                            <tr><th>Type</th><th>Name</th><th>Created</th><th>Last modified</th><th>Size</th></tr>\r\n\
-                                            <tr><th colspan=\"4\"><hr></th></tr>\r\n")
+                                            <tr><th colspan=\"5\"><hr></th></tr>\r\n")
-                                            , client->_httpRequestData.httpResource, client->_httpRequestData.httpResource);*/
+                                            , client->_httpRequestData.httpResource, client->_httpRequestData.httpResource);
-      //File nextFile;
+      
+       if(strlen(client->_httpRequestData.httpResource) > 1) //Then we are not at the root of the WEBServer's directory.
+       {
+         char *rsrcCopy = strdup(client->_httpRequestData.httpResource); 
+         if(rsrcCopy)
+         {
+            char *lastSlash(strrchr(rsrcCopy, '/'));
+            if(lastSlash)*lastSlash = '\0';
+            lastSlash = strrchr(rsrcCopy, '/');
+            if(lastSlash)*lastSlash = '\0';
+
+            client->_client.printf_P(PSTR("<tr><td>[DIR]</td><td><a href=\"%s/\">Parent Directory</a></td><td> - </td><td align=\"right\"> - </td><td> </td></tr>\r\n"), rsrcCopy);
+            free(rsrcCopy);
+         }
+       }
+
+      File nextFile;
       for(;;)
       {
-        //if(!(nextFile = pageToSend.openNextFile()))break;
+        if(!(nextFile = pageToSend.openNextFile()))break;
-        File nextFile = pageToSend.openNextFile();
-        if (!nextFile) //No more files in the directory
-              break;
-        //time_t creationTime = nextFile.getCreationTime(), lastModified = nextFile.getLastWrite();
         
-        /*client->_client.printf_P(PSTR("<tr><td><a href=\"%s\">%s</a></td><td align=\"right\">2021-07-09 15:37  </td><td align=\"right\">2021-07-09 15:37  </td><td align=\"right\"> %u</td></tr>\r\n"),
+        char zero_prepended[4][3] = {"","","",""};
+        time_t rawCreationTime(nextFile.getCreationTime()), rawLastModifiedTime(nextFile.getLastWrite());
+        tm creationTime(*localtime(&rawCreationTime)), lastModifiedTime(*localtime(&rawLastModifiedTime));
+
+        client->_client.printf_P(PSTR("<tr><td>%s</td><td><a href=\"%s\">%s</a></td><td align=\"right\">%d-%d-%d %s:%s</td><td align=\"right\">%d-%d-%d %s:%s</td><td align=\"right\">%.1fK</td></tr>\r\n"),
+        nextFile.isDirectory() ? "[DIR]":"[FILE]",
         nextFile.name(),
         nextFile.name(),
-        nextFile.size()
+        creationTime.tm_year + 1900, creationTime.tm_mon + 1, creationTime.tm_mday, dateTimeFormater(zero_prepended[0], creationTime.tm_hour, '0'), dateTimeFormater(zero_prepended[1], creationTime.tm_min, '0'),
-        );*/
+        lastModifiedTime.tm_year + 1900, lastModifiedTime.tm_mon + 1, lastModifiedTime.tm_mday, dateTimeFormater(zero_prepended[2], lastModifiedTime.tm_hour, '0'), dateTimeFormater(zero_prepended[3], lastModifiedTime.tm_min, '0'),
-        Serial.printf("%s %u\r\n", nextFile.name(), nextFile.size());
+        nextFile.size() / 1024.0
+        );
+
+        #ifdef DEBUG_WEBS
+        Serial.printf("File name : %s\nFile size : %u\nFree stack : %u\n", nextFile.name(), nextFile.size(), ESP.getFreeContStack());
+        #endif
         nextFile.close();
       }
 
-      /*client->_client.printf_P(PSTR(       "<tr><th colspan=\"4\"><hr></th></tr>\r\n\
+      client->_client.printf_P(PSTR(       "<tr><th colspan=\"5\"><hr></th></tr>\r\n\
                                         </table>\r\n\
                                         <address>SAB WEBServer, Version %s at %s Port %u</address>\r\n\
                                       </body>\r\n\
-                                    </html>"), "1.0.0", "", TCPServer<T>::getPort()); //TODO get IP*/
+                                    </html>"), "1.0.0", client->_client.localIP().toString().c_str(), TCPServer<T>::getPort());
     }
     
     /*Static helper methods*/
 
-    static void sendInfoResponse(HTTP_CODE http_code, T *client, const char *message)
+    static void sendInfoResponse(HTTP_CODE http_code, T *client, const char *message, const char *location = nullptr)
     {
       char codeLiteral[100];
       switch(http_code)
       {
+        case HTTP_CODE_MOVED_PERMANENTLY:
+          strcpy_P(codeLiteral,PSTR("Moved Permanently"));
+          break;
         case HTTP_CODE_BAD_REQUEST:
           strcpy_P(codeLiteral,PSTR("Bad Request"));
           break;
@@ -768,7 +829,13 @@ class WEBServer : public TCPServer<T>, public HttpConstants
           strcpy_P(codeLiteral,PSTR("Error Not Defined"));
           break;
       }
-      client->_client.printf_P(PSTR("HTTP/1.1 %d %s\r\nContent-Type: text/html\r\nContent-Length: %d\r\n\r\n<!DOCTYPE HTML>\r\n<html>\r\n<h1>Error %d</h1><p>%s</p>\r\n</html>"), http_code, codeLiteral, strlen(message) + 56 + (http_code != 0 ? 3:1), http_code , message);
+      client->_client.printf_P(PSTR("HTTP/1.1 %d %s\r\n"), http_code, codeLiteral);
+
+
+      if(http_code == HTTP_CODE_MOVED_PERMANENTLY)
+        client->_client.printf_P(PSTR("Location: %s\r\n"), location);
+
+      client->_client.printf_P(PSTR("Content-Type: text/html\r\nContent-Length: %d\r\n\r\n<!DOCTYPE HTML>\r\n<html>\r\n<h1>Error %d</h1><p>%s</p>\r\n</html>"), strlen(message) + 56 + (http_code != 0 ? 3:1), http_code , message);
     }
     
 		static HttpRequestMethod getHttpVerbEnumValue(const char *parseBuffer)
@@ -802,6 +869,7 @@ class WEBServer : public TCPServer<T>, public HttpConstants
       else if(strcmp(extension, "jpg") == 0) return IMAGE_JPEG;
       else if(strcmp(extension, "ico") == 0) return IMAGE_X_ICO;
       else if(strcmp(extension, "mp3") == 0) return AUDIO_MPEG;
+      else if(strcmp(extension, "txt") == 0) return TEXT_PLAIN;
       else return UNKNOWN_MIME;
     }
     

src/app/tasks.cpp

@@ -17,11 +17,12 @@ boolean task_sys_info(void *pData)
   uint8_t HEAPfrag;
   ESP.getHeapStats(&freeRAM, &biggestContigMemBlock, &HEAPfrag);
 
-  Serial.printf_P(PSTR("BATT SENSING...\nRunning since : %d s\nSystem error : %u\nIP : %s, %d dBm\nWEB Server clients : %u\nMemory info:\n\tFree RAM : %u\n\tHeap frag : %u\n\tMax block %u\n"),
+  Serial.printf_P(PSTR("BATT SENSING...\nRunning since : %d s\nSystem error : %u\nStation IP : %s, %d dBm\nAP IP : %s\nWEB Server clients : %u\nMemory info:\n\tFree RAM : %u\n\tHeap frag : %u\n\tMax block %u\n"),
   millis()/1000,
   p->sab->getError(),
   p->sab->getConnectivityManager().localIP().toString().c_str(),
   p->sab->getConnectivityManager().RSSI(),
+  p->sab->getConnectivityManager().softAPIP().toString().c_str(),
   p->sab->getWebServer().getConnectedClientsCount(),
   freeRAM,
   HEAPfrag,

src/app/utilities.cpp

@@ -68,3 +68,20 @@ uint32_t monthNumTo3LetterAbbreviation(const uint8_t monthNumber)
     
   return toReturn;
 }
+
+void *mallocWithFallback(size_t requestedSize, size_t *acceptedSize)
+{
+  void *allocAddr(nullptr);
+  uint32_t divider(1);
+  
+  do
+  {
+    requestedSize /= divider++;
+    allocAddr = malloc(requestedSize);
+  } while(!allocAddr && requestedSize >= 50); //If we can't even manage to malloc 50 bytes then we should give up and cry
+
+  if(acceptedSize && allocAddr)*acceptedSize = requestedSize;
+  else if(acceptedSize)*acceptedSize = 0;
+
+  return allocAddr;
+}

src/app/utilities.h

@@ -5,4 +5,5 @@
 char *addChar(char *pointer, const char character);
 char *dateTimeFormater(char *pointer, const uint8_t value, const char character);
 uint32_t monthNumTo3LetterAbbreviation(const uint8_t monthNumber);
+void *mallocWithFallback(size_t requestedSize, size_t *acceptedSize = nullptr);
 #endif // UTILITIES_H